Integrate CertFetcher with flag-protection into AMP Packager.

cmd/amppkg/main.go

@@ -40,6 +40,9 @@ var flagConfig = flag.String("config", "amppkg.toml", "Path to the config toml f
 var flagDevelopment = flag.Bool("development", false, "True if this is a development server.")
 var flagInvalidCert = flag.Bool("invalidcert", false, "True if invalid certificate intentionally used in production.")
 
+// IMPORTANT: do not turn on this flag for now, it's still under development.
+var flagAutoRenewCert = flag.Bool("autorenewcert", false, "True if amppackager is to attempt cert auto-renewal.")
+
 // Prints errors returned by pkg/errors with stack traces.
 func die(err interface{}) { log.Fatalf("%+v", err) }
 
@@ -82,7 +85,7 @@ func main() {
 		die(errors.Wrap(err, "loading key file"))
 	}
 
-	certCache, err := certloader.PopulateCertCache(config, key, *flagDevelopment || *flagInvalidCert);
+	certCache, err := certloader.PopulateCertCache(config, key, *flagDevelopment || *flagInvalidCert, *flagAutoRenewCert)
 	if err != nil {
 		die(errors.Wrap(err, "building cert cache"))
 	}

cmd/gateway_server/go.mod

@@ -0,0 +1,11 @@
+module github.com/ampproject/amppackager/cmd/gateway_server
+
+go 1.13
+
+require (
+	github.com/WICG/webpackage v0.0.0-20190904045429-ab0e18a2113c // indirect
+	github.com/ampproject/amppackager v0.0.0-20190911170541-c9993b8ac4d1 // indirect
+	github.com/golang/protobuf v1.3.2 // indirect
+	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect
+	google.golang.org/grpc v1.23.1 // indirect
+)

cmd/gateway_server/go.sum

@@ -0,0 +1,45 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/WICG/webpackage v0.0.0-20190904045429-ab0e18a2113c h1:z+hJKXGCAdk9H/JKAmACY2bKyLhAInkjMh/VVlKNxEI=
+github.com/WICG/webpackage v0.0.0-20190904045429-ab0e18a2113c/go.mod h1:RmCSqjSsrBtTrYiO+OKMBfMgztnTnFXVSapi/mNB7IA=
+github.com/ampproject/amppackager v0.0.0-20190911170541-c9993b8ac4d1 h1:vwqF8hQdUaCP6u9FzVXwwFKJ+5kdQDOkp2vYlTbTSGE=
+github.com/ampproject/amppackager v0.0.0-20190911170541-c9993b8ac4d1/go.mod h1:kqUX/cvxB07nOtlh8/35T4sXNOos7/kqOJTN34XAvF8=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/influxdata/influxdb v1.6.3/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/mrichman/hargo v0.1.2-0.20190117125451-162adce4527e/go.mod h1:ycD51zRGXcO6ak4DnFPjHv4xzbgRU5tYyWDzbMzFYKw=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190110200230-915654e7eabc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

cmd/gateway_server/server.go

@@ -74,7 +74,7 @@ func (s *gatewayServer) GenerateSXG(ctx context.Context, request *pb.SXGRequest)
 	}
 
 	// Note: do not initialize certCache, we just want it to hold the certs for now.
-	certCache := certcache.New(certs, "");
+	certCache := certcache.New(certs, nil, "");
 
 	privateKey, err := util.ParsePrivateKey(request.PrivateKey)
 	if err != nil {

packager/certcache/certcache.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/WICG/webpackage/go/signedexchange/certurl"
+	"github.com/ampproject/amppackager/packager/certfetcher"
 	"github.com/ampproject/amppackager/packager/mux"
 	"github.com/ampproject/amppackager/packager/util"
 	"github.com/pkg/errors"
@@ -46,10 +47,19 @@ var infiniteFuture = time.Unix(1<<63-62135596801, 999999999)
 // 1MB is the maximum used by github.com/xenolf/lego/acmev2 in GetOCSPForCert.
 // Alternatively, here's a random documented example of a 20K limit:
 // https://www.ibm.com/support/knowledgecenter/en/SSPREK_9.0.0/com.ibm.isam.doc/wrp_stza_ref/reference/ref_ocsp_max_size.html
-const maxOCSPResponseBytes = 1024 * 1024
+const MaxOCSPResponseBytes = 1024 * 1024
 
 // How often to check if OCSP stapling needs updating.
-const ocspCheckInterval = 1 * time.Hour
+const OcspCheckInterval = 1 * time.Hour
+
+// How often to check if certs needs updating.
+const CertCheckInterval = 24 * time.Hour
+
+// Recommended renewal duration for certs.
+// 8 days is recommended duration to start requesting new certs to allow for ACME server outages.
+// It's 6 days + 2 days renewal grace period.
+// TODO(banaag): make 2 days renewal grace period configurable in toml.
+const CertRenewalInterval = 8 * 24 * time.Hour
 
 type CertHandler interface {
 	GetLatestCert() (*x509.Certificate)
@@ -58,25 +68,35 @@ type CertHandler interface {
 type CertCache struct {
 	// TODO(twifkak): Support multiple cert chains (for different domains, for different roots).
 	certName          string
+	certsMu		  sync.RWMutex
 	certs             []*x509.Certificate
+	// If certFetcher is not set, that means cert auto-renewal is not available.
+	certFetcher	  *certfetcher.CertFetcher
+	renewedCertsMu	  sync.RWMutex
+	renewedCerts	  []*x509.Certificate
 	ocspUpdateAfterMu sync.RWMutex
 	ocspUpdateAfter   time.Time
 	// TODO(twifkak): Implement a registry of Updateable instances which can be configured in the toml.
-	ocspFile Updateable
-	client   http.Client
+	ocspFile	  Updateable
+	client		  http.Client
 
 	// "Virtual methods", exposed for testing.
 	// Given a certificate, returns the OCSP responder URL for that cert.
 	extractOCSPServer func(*x509.Certificate) (string, error)
 	// Given an HTTP request/response, returns its cache expiry.
 	httpExpiry func(*http.Request, *http.Response) time.Time
+	// Is CertCache initialized to do cert renewal or OCSP refreshes?
+	isInitialized	  bool
 }
 
-// Must call Init() on the returned CertCache before you can use it.
-func New(certs []*x509.Certificate, ocspCache string) *CertCache {
+// Callers need to call Init() on the returned CertCache before the cache can auto-renew certs.
+// Callers can use the uninitialized CertCache either for testing certificates (without doing OCSP or
+// cert refreshes).
+func New(certs []*x509.Certificate, certFetcher *certfetcher.CertFetcher, ocspCache string) *CertCache {
 	return &CertCache{
 		certName:        util.CertName(certs[0]),
 		certs:           certs,
+		certFetcher:	 certFetcher,
 		ocspUpdateAfter: infiniteFuture, // Default, in case initial readOCSP successfully loads from disk.
 		// Distributed OCSP cache to support the following sleevi requirements:
 		// 1. Support for keeping a long-lived (disk) cache of OCSP responses.
@@ -104,6 +124,7 @@ func New(certs []*x509.Certificate, ocspCache string) *CertCache {
 				return expiry
 			}
 		},
+		isInitialized: false,
 	}
 }
 
@@ -122,15 +143,40 @@ func (this *CertCache) Init(stop chan struct{}) error {
 	//    to raise an alert if something has gone really wrong.
 	// 7. The ability to serve old responses while fetching new responses.
 	go this.maintainOCSP(stop)
+
+	if this.certFetcher != nil {
+		// Update Certs in the background.
+		go this.maintainCerts(stop)
+	}
+
+	this.isInitialized = true
 	return nil
 }
 
 // For now this just returns the first entry in the certs field in the cache.
 // For follow-on changes, we will transform this to a lambda so that anything
 // that needs a cert can do the cert refresh logic (if needed)  on demand.
 func (this *CertCache) GetLatestCert() (*x509.Certificate) {
-	// TODO(banaag): check if cert is still valid, refresh if not.
-	return this.certs[0]
+	if !this.isInitialized || this.certFetcher == nil {
+		// If certcache is not initialized or certFetcher is not set,
+		// just return cert without checking if it needs auto-renewal.
+		return this.certs[0]
+	}
+	d, err := util.GetDurationToExpiry(this.certs[0], this.certs[0].NotAfter)
+	if err != nil {
+		// Current cert is already invalid. Check if renewal is available.
+		log.Println("Current cert is expired, attempting to renew.")
+		return nil
+	}
+	if d >= time.Duration(CertRenewalInterval) {
+		// Cert is still valid.
+		return this.certs[0]
+	} else if d < time.Duration(CertRenewalInterval) {
+		// Cert is still valid, but we need to start process of requesting new cert.
+		log.Println("Current cert is close to expiry threshold, attempting to renew in the background.")
+		return this.certs[0]
+	}
+	return nil
 }
 
 func (this *CertCache) createCertChainCBOR(ocsp []byte) ([]byte, error) {
@@ -263,7 +309,7 @@ func (this *CertCache) maintainOCSP(stop chan struct{}) {
 	//    has trouble getting a request, hopefully it does something
 	//    smarter than just retry in a busy loop, hammering the OCSP server
 	//    into further oblivion.
-	ticker := time.NewTicker(ocspCheckInterval)
+	ticker := time.NewTicker(OcspCheckInterval)
 
 	for {
 		select {
@@ -440,3 +486,77 @@ func (this *CertCache) fetchOCSP(orig []byte, ocspUpdateAfter *time.Time) []byte
 	}
 	return respBytes
 }
+
+
+// Checks for cert updates every certCheckInterval hours. Never terminates.
+func (this *CertCache) maintainCerts(stop chan struct{}) {
+	// Only make one request per certCheckInterval, to minimize the impact
+	// on servers that are buckling under load.
+	ticker := time.NewTicker(CertCheckInterval)
+
+	for {
+		select {
+		case <-ticker.C:
+			this.updateCertIfNecessary()
+		case <-stop:
+			ticker.Stop()
+			return
+		}
+	}
+}
+
+// Set current cert with mutex protection.
+func (this *CertCache) setCerts(certs []*x509.Certificate) {
+	this.certsMu.Lock()
+	defer this.certsMu.Unlock()
+	this.certs = certs
+}
+
+// Set new cert with mutex protection.
+func (this *CertCache) setNewCerts(certs []*x509.Certificate) {
+	this.renewedCertsMu.Lock()
+	defer this.renewedCertsMu.Unlock()
+	this.renewedCerts = certs
+}
+
+// Update the cert in the cache if necessary.
+func (this *CertCache) updateCertIfNecessary() {
+	if this.certFetcher == nil {
+		// Do nothing if certFetcher is not set.
+		return
+	}
+	d, err := util.GetDurationToExpiry(this.certs[0], this.certs[0].NotAfter)
+	if err != nil {
+		if this.renewedCerts != nil {
+			// If renewedCerts is set, copy that over to certs
+			// and set renewedCerts to nil.
+			// TODO(banaag): do the same cert setting dance on disk.
+			// Purge OCSP cache? Make shouldUpdateOCSP() return true?
+			this.setCerts(this.renewedCerts)
+			this.setNewCerts(nil)
+			return
+		}
+		// Current cert is already invalid. Try refreshing.
+		log.Println("Warning current cert is expired, attempting to renew: ", err)
+		certs, err := this.certFetcher.FetchNewCert()
+		if err != nil {
+			log.Println("Error trying to fetch new certificates from CA: ", err)
+			return
+		}
+		this.setCerts(certs)
+		return
+	}
+	if d >= time.Duration(CertRenewalInterval) {
+		// Cert is still valid, don't do anything.
+	} else if d < time.Duration(CertRenewalInterval) {
+		// Cert is still valid, but we need to start process of requesting new cert.
+		log.Println("Warning: Current cert crossed threshold for renewal, attempting to renew.")
+		certs, err := this.certFetcher.FetchNewCert()
+		if err != nil {
+			log.Println("Error trying to fetch new certificates from CA: ", err)
+			return
+		}
+		// TODO(banaag): save this cert to disk.
+		this.setNewCerts(certs)
+	}
+}

packager/certcache/certcache_test.go

@@ -74,7 +74,9 @@ type CertCacheSuite struct {
 
 func (this *CertCacheSuite) New() (*CertCache, error) {
 	// TODO(twifkak): Stop the old CertCache's goroutine.
-	certCache := New(pkgt.B3Certs, filepath.Join(this.tempDir, "ocsp"))
+	// TODO(banaag): Consider adding a test with certfetcher set.
+	//  For now, this tests certcache without worrying about certfetcher.
+	certCache := New(pkgt.B3Certs, nil, filepath.Join(this.tempDir, "ocsp"))
 	certCache.extractOCSPServer = func(*x509.Certificate) (string, error) {
 		return this.ocspServer.URL, nil
 	}

packager/certloader/certloader.go

@@ -24,24 +24,48 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/ampproject/amppackager/packager/certcache"
+	"github.com/ampproject/amppackager/packager/certfetcher"
 	"github.com/ampproject/amppackager/packager/util"
 )
 
 // Creates cert cache by loading certs and keys from disk, doing validation
 // and populating the cert cache with current set of certificate related information.
 // If development mode is true, prints a warning for certs that can't sign HTTP exchanges.
-func PopulateCertCache(config *util.Config, key crypto.PrivateKey, developmentMode bool) (*certcache.CertCache, error) {
+func PopulateCertCache(config *util.Config, key crypto.PrivateKey,
+	developmentMode bool, autoRenewCert bool) (*certcache.CertCache, error) {
+
 	certs, err := loadCertsFromFile(config, developmentMode)
 	if err != nil {
 		return nil, err
 	}
+	domain := ""
 	for _, urlSet := range config.URLSet {
-		domain := urlSet.Sign.Domain
+		domain = urlSet.Sign.Domain
 		if err := util.CertificateMatches(certs[0], key, domain); err != nil {
 			return nil, errors.Wrapf(err, "checking %s", config.CertFile)
 		}
 	}
-	certCache := certcache.New(certs, config.OCSPCache)
+
+	certFetcher := (*certfetcher.CertFetcher)(nil)
+	if autoRenewCert {
+		emailAddress := config.ACMEConfig.Production.EmailAddress
+		acmeDiscoveryURL := config.ACMEConfig.Production.DiscoURL
+		challengePort := config.ACMEConfig.Production.ChallengePort
+		if developmentMode {
+			emailAddress = config.ACMEConfig.Development.EmailAddress
+			acmeDiscoveryURL = config.ACMEConfig.Development.DiscoURL
+			challengePort = config.ACMEConfig.Development.ChallengePort
+		}
+
+		// Create the cert fetcher that will auto-renew the cert.
+		certFetcher, err = certfetcher.NewFetcher(emailAddress, key, acmeDiscoveryURL,
+			[]string{domain}, challengePort, !developmentMode)
+		if err != nil {
+			return nil, errors.Wrap(err, "creating certfetcher")
+		}
+	}
+
+	certCache := certcache.New(certs, certFetcher, config.OCSPCache)
 
 	return certCache, nil
 }

packager/certloader/certloader_test.go

@@ -59,10 +59,11 @@ func TestPopulateCertCache(t *testing.T) {
 			}},
 		},
 		pkgt.B3Key,
-		true)
+		true,
+		false)
+	assert.Nil(t, err)
 	assert.NotNil(t, certCache)
 	assert.Equal(t, pkgt.B3Certs[0], certCache.GetLatestCert())
-	assert.Nil(t, err)
 }
 
 func TestLoadCertsFromFile(t *testing.T) {

packager/util/config.go

@@ -68,6 +68,8 @@ type ACMEServerConfig struct {
 	DiscoURL	string // ACME Directory Resource URL
 	AccountURL	string // ACME Account URL. If non-empty, we
 			       // will auto-renew cert via ACME.
+	EmailAddress	string // Email address registered with ACME CA.
+	ChallengePort	int    // ACME challenge port.
 }
 
 // TODO(twifkak): Extract default values into a function separate from the one