diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 62a5aea36..000000000 --- a/Gopkg.lock +++ /dev/null @@ -1,134 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - branch = "master" - name = "github.com/WICG/webpackage" - packages = [ - "go/signedexchange", - "go/signedexchange/cbor", - "go/signedexchange/certurl", - "go/signedexchange/internal/bigendian", - "go/signedexchange/internal/signingalgorithm", - "go/signedexchange/mice", - "go/signedexchange/structuredheader", - "go/signedexchange/version" - ] - revision = "70386c3750f2ba5fbcb6e66835df0578b2a55c13" - -[[projects]] - branch = "master" - name = "github.com/ampproject/amphtml" - packages = ["validator"] - revision = "d3df64d07ae9e9e21a94d8339f3c2d0a9b7d4261" - -[[projects]] - name = "github.com/davecgh/go-spew" - packages = ["spew"] - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" - -[[projects]] - name = "github.com/gofrs/flock" - packages = ["."] - revision = "392e7fae8f1b0bdbd67dad7237d23f618feb6dbb" - version = "v0.7.1" - -[[projects]] - name = "github.com/golang/protobuf" - packages = ["proto"] - revision = "c823c79ea1570fb5ff454033735a8e68575d1d0f" - version = "v1.3.0" - -[[projects]] - name = "github.com/google/go-cmp" - packages = [ - "cmp", - "cmp/cmpopts", - "cmp/internal/diff", - "cmp/internal/function", - "cmp/internal/value" - ] - revision = "3af367b6b30c263d47e8895973edcca9a49cf029" - version = "v0.2.0" - -[[projects]] - name = "github.com/pelletier/go-toml" - packages = ["."] - revision = "acdc4509485b587f5e675510c4f2c63e90ff68a8" - version = "v1.1.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - name = "github.com/pmezard/go-difflib" - packages = ["difflib"] - revision = "792786c7400a136282c1664665ae0a8db921c6c2" - version = "v1.0.0" - -[[projects]] - branch = "master" - name = "github.com/pquerna/cachecontrol" - packages = [ - ".", - "cacheobject" - ] - revision = "525d0eb5f91d30e3b1548de401b7ef9ea6898520" - -[[projects]] - name = "github.com/stretchr/testify" - packages = [ - "assert", - "require", - "suite" - ] - revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71" - version = "v1.2.1" - -[[projects]] - branch = "master" - name = "golang.org/x/crypto" - packages = ["ocsp"] - revision = "614d502a4dac94afa3a6ce146bd1736da82514c6" - -[[projects]] - branch = "master" - name = "golang.org/x/net" - packages = [ - "html", - "html/atom", - "idna" - ] - revision = "f9ce57c11b242f0f1599cf25c89d8cb02c45295a" - -[[projects]] - name = "golang.org/x/text" - packages = [ - "collate", - "collate/build", - "internal/colltab", - "internal/gen", - "internal/tag", - "internal/triegen", - "internal/ucd", - "language", - "secure/bidirule", - "transform", - "unicode/bidi", - "unicode/cldr", - "unicode/norm", - "unicode/rangetable" - ] - revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" - version = "v0.3.0" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "10bd66ddd0a417265606dbeca1aa951d8bc4040b79b1bc00c4b9a107d8b2fdf7" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index c21e72dbb..000000000 --- a/Gopkg.toml +++ /dev/null @@ -1,33 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - - -ignored = ["google.golang.org/grpc"] - -[prune] - go-tests = true - unused-packages = true - non-go = true diff --git a/README.md b/README.md index 2abd069a9..0aeea4ee4 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ own and can obtain certificates for. 1. Install Go version 1.10 or higher. Optionally, set [$GOPATH](https://github.com/golang/go/wiki/GOPATH) to something (default is `~/go`) and/or add `$GOPATH/bin` to `$PATH`. - 2. `go get -u github.com/ampproject/amppackager/cmd/amppkg` + 2. `go get -u -mod=vendor github.com/ampproject/amppackager/cmd/amppkg` Optionally, move the built `~/go/bin/amppkg` wherever you like. 3. Create a file `amppkg.toml`. A minimal config looks like this: @@ -78,7 +78,7 @@ container. #### Demonstrate privacy-preserving prefetch This step is optional; just to show how [privacy-preserving -prefetch](https://wicg.github.io/webpackage/draft-yasskin-webpackage-use-cases.html#private-prefetch) +prefetch](https://wicg.github.io/webpackage/draft-yasskin-wpack-use-cases.html#private-prefetch) works with SXGs. 1. `go get -u github.com/ampproject/amppackager/cmd/amppkg_dl_sxg`. @@ -131,7 +131,9 @@ For now, productionizing is a bit manual. The minimum steps are: team will release a new version approximately this often. Soon after each release, Googlebot will increment the version it requests with `AMP-Cache-Transform`. Googlebot will only allow the latest 2-3 versions - (details are still TBD), so an update is necessary but not immediately. + (details are still TBD), so an update is necessary but not immediately. If + amppkg doesn't support the requested version range, it will fall back to + serving unsigned AMP. To keep subscribed to releases, you can select "Releases only" from the "Watch" dropdown in GitHub, or use [various tools](https://stackoverflow.com/questions/9845655/how-do-i-get-notifications-for-commits-to-a-repository) diff --git a/amppkg.example.toml b/amppkg.example.toml index 2ac6f8872..2431bcc3a 100644 --- a/amppkg.example.toml +++ b/amppkg.example.toml @@ -47,6 +47,26 @@ # SHA-256). CertFile = './pems/cert.pem' +# The path to save a new cert retrieved from the CA if the current cert in +# 'CertFile' above is still valid. +# This is optional and is needed only if you have 'autorenewcert' turned on. +# For multi-replica setups (multiple AMP Packager instances), only the replica +# that will do the autorenewal of certs needs this config item set. +# NewCertFile = './pems/newcert.pem' + +# The path to the Certificate Signing Request (CSR) that is needed to request +# new certificates from the Certificate Authority using ACME. +# CSRs are typically created using the openssl command: +# openssl req -new -key /path/to/privkey -out /path/to/cert.csr +# To verify: +# openssl req -text -noout -verify -in cert.csr +# The following docs list examples on how to go about generating CSRs: +# https://www.digicert.com/csr-creation.htm?rid=011592 +# https://www.ssl.com/how-to/manually-generate-a-certificate-signing-request-csr-using-openssl/ +# https://geekflare.com/san-ssl-certificate/ +# This is optional and is needed only if you have 'autorenewcert' turned on. +# CSRFile = './pems/cert.csr' + # The path to the PEM file containing the private key that corresponds to the # leaf certificate in CertFile. KeyFile = './pems/privkey.pem' @@ -170,3 +190,81 @@ ForwardedRequestHeaders = [] # Domain = "www.corp.amppackageexample.com" # PathRE = "/world/.*" # QueryRE = "" + +# IMPORTANT NOTE: the support of the ACME protocol and automatic renewal of certificates is currently in the +# EXPERIMENTAL stage. Once we have more experience with people using it out in the wild, we will gradually +# move it to PRODUCTION mode. +# +# ACME is a protocol that allows for automatic renewal of certificates. AMP Packager uses an ACME library +# https://github.com/go-acme/lego to handle certificate renewal. Automatic certificate renewal is enabled +# in AMP Packager via the 'autorenewcert' flag. Turning the flag on will enable AMP Packager to automatically +# request certificate renewals whenever it has determined that the current certificate is expired or about to +# expire. +# +# ACMEConfig only needs to be present in the toml file if 'autorenewcert' command line flag was turned on. +# If the flag is on, at least one of ACMEConfig.Production or ACMEConfig.Development should be present. +# Note that a recommended best practice for setting up the cert renewal that minimizes both cost and bombarding +# your Certificate Authority with requests is that for a multi-instance setup of AMP packager, only one instance is +# setup to do automatic cert renewals and the rest of the instances will just be configured to reload the fresh +# certificate from disk when their in-memory copies expire. This also implies that the cert paths configured above +# in 'CertFile' and 'NewCertFile' are located on a shared filesystem accessible by all AMP packager instances. +# +# For the full ACME spec, see: +# https://tools.ietf.org/html/draft-ietf-acme-acme-02 +# https://ietf-wg-acme.github.io/acme/draft-ietf-acme-acme.html +# TODO(banaag): consider renaming ACMEConfig to ACME +# [ACMEConfig] + # This config will be used if 'autorenewcert' is turned on and 'development' is turned off. + # If the flags above are on but we don't have an entry here, AMP Packager will not start. + # [ACMEConfig.Production] + # This is the ACME discovery URL that is used for ACME http requests to the Certificate Authority that + # doles out the certificates. + # Currently, the only CA that supports automatic signed exchange cert renewals is Digicert: + # https://docs.digicert.com/certificate-tools/acme-user-guide/acme-directory-urls-signed-http-exchange-certificates/ + # DiscoURL = "https://production-acme.discovery.url/" + + # This is the email address you used to create an account with the Certificate Authority that is registered to + # request signed exchange certificates. + # EmailAddress = "user@company.com" + + # For the remaining configuration items, it's important to understand the different challenges employed as + # part of the ACME protocol. See: + # https://ietf-wg-acme.github.io/acme/draft-ietf-acme-acme.html#identifier-validation-challenges + # https://letsencrypt.org/docs/challenge-types/ + # https://certbot.eff.org/docs/challenges.html?highlight=http + # Note that you don't need to have all the challenges configured, it's typically sufficient to have one configured. + # The exception arises when you have to deal with wildcard certificates, see below. + + # This is the http server root directory where the ACME http challenge token could be deposited. Note that you may + # need to do some configuration work to get this setup to work where multiple instances of AMP Packager is running. + # For example: + # https://community.letsencrypt.org/t/how-to-nginx-configuration-to-enable-acme-challenge-support-on-all-http-virtual-hosts/5622/3 + # HttpWebRootDir = '/path/to/www_root_dir' + + # This is the port used by the AMP Packager to respond to the HTTP challenge issued as part of ACME protocol. + # Note that if your setup only opens up certain ports, you may need to do a configuration change where you forward + # requests to this port using proxy_pass, for example: + # https://medium.com/@dipeshwagle/add-https-using-lets-encrypt-to-nginx-configured-as-a-reverse-proxy-on-ubuntu-b4455a729176 + # HttpChallengePort = 5002 + + # This is the port used by AMP packager to respond to the TLS challenge issued as part of the ACME protocol. + # TlsChallengePort = 5003 + + # This is the DnsProvider to be used in fulfilling the ACME DNS challenge. Note that you only need the DNS challenge + # setup if you have wildcard certificates. See: https://searchsecurity.techtarget.com/definition/wildcard-certificate + # For the DNS challenge, go-acme/lego, there are certain environment variables that need to be set up which depends on + # the DNS provider that you use to fulfill the DNS challenge. See: + # https://go-acme.github.io/lego/dns/ + # DnsProvider = "gcloud" + + # This config will be used if 'autorenewcert' is turned on and 'development' is turned on. + # If the flags above are on but we don't have an entry here, AMP Packager will not start. + # All the other fields below have the same semantics as the one in ACMEConfig.Production above. + # For development mode, given that we don't require the SXG extension, one can use Let's Encrypt CA to generate the certs. + # [ACMEConfig.Development] + # DiscoURL = "https://development-acme.discovery.url/" + # EmailAddress = "user@company.com" + # HttpChallengePort = 5002 + # HttpWebRootDir = '/path/to/www_root_dir' + # TlsChallengePort = 5003 + # DnsProvider = "gcloud" diff --git a/cmd/amppkg/main.go b/cmd/amppkg/main.go index 2208dc97c..d5e8a5bb9 100644 --- a/cmd/amppkg/main.go +++ b/cmd/amppkg/main.go @@ -18,6 +18,7 @@ package main import ( + "crypto/ecdsa" "flag" "fmt" "io/ioutil" @@ -26,10 +27,11 @@ import ( "net/url" "time" - "github.com/WICG/webpackage/go/signedexchange" "github.com/pkg/errors" "github.com/ampproject/amppackager/packager/certcache" + "github.com/ampproject/amppackager/packager/certloader" + "github.com/ampproject/amppackager/packager/healthz" "github.com/ampproject/amppackager/packager/mux" "github.com/ampproject/amppackager/packager/rtv" "github.com/ampproject/amppackager/packager/signer" @@ -41,6 +43,9 @@ var flagConfig = flag.String("config", "amppkg.toml", "Path to the config toml f var flagDevelopment = flag.Bool("development", false, "True if this is a development server.") var flagInvalidCert = flag.Bool("invalidcert", false, "True if invalid certificate intentionally used in production.") +// IMPORTANT: do not turn on this flag for now, it's still under development. +var flagAutoRenewCert = flag.Bool("autorenewcert", false, "True if amppackager is to attempt cert auto-renewal.") + // Prints errors returned by pkg/errors with stack traces. func die(err interface{}) { log.Fatalf("%+v", err) } @@ -73,52 +78,39 @@ func main() { die(errors.Wrapf(err, "parsing config at %s", *flagConfig)) } - // TODO(twifkak): Document what cert/key storage formats this accepts. - certPem, err := ioutil.ReadFile(config.CertFile) - if err != nil { - die(errors.Wrapf(err, "reading %s", config.CertFile)) - } - keyPem, err := ioutil.ReadFile(config.KeyFile) + validityMap, err := validitymap.New() if err != nil { - die(errors.Wrapf(err, "reading %s", config.KeyFile)) + die(errors.Wrap(err, "building validity map")) } - certs, err := signedexchange.ParseCertificates(certPem) + key, err := certloader.LoadKeyFromFile(config) if err != nil { - die(errors.Wrapf(err, "parsing %s", config.CertFile)) - } - if certs == nil || len(certs) == 0 { - die(fmt.Sprintf("no cert found in %s", config.CertFile)) - } - if err := util.CanSignHttpExchanges(certs[0], time.Now()); err != nil { - if *flagDevelopment || *flagInvalidCert { - log.Println("WARNING:", err) - } else { - die(err) - } + die(errors.Wrap(err, "loading key file")) } - key, err := util.ParsePrivateKey(keyPem) + var responder certcache.OCSPResponder = nil + if *flagDevelopment { + // Key is guaranteed to be ECDSA by signedexchange.ParsePrivateKey. This may change in future versions of SXG. + responder = fakeOCSPResponder{key: key.(*ecdsa.PrivateKey)}.Respond + } + certCache, err := certcache.PopulateCertCache(config, key, responder, *flagDevelopment || *flagInvalidCert, *flagAutoRenewCert) if err != nil { - die(errors.Wrapf(err, "parsing %s", config.KeyFile)) + die(errors.Wrap(err, "building cert cache")) } - for _, urlSet := range config.URLSet { - domain := urlSet.Sign.Domain - if err := util.CertificateMatches(certs[0], key, domain); err != nil { - die(errors.Wrapf(err, "checking %s", config.CertFile)) + if err = certCache.Init(); err != nil { + if *flagDevelopment { + fmt.Println("WARNING:", err) + } else { + die(errors.Wrap(err, "initializing cert cache")) } - } + } - validityMap, err := validitymap.New() + healthz, err := healthz.New(certCache) if err != nil { - die(errors.Wrap(err, "building validity map")) + die(errors.Wrap(err, "building healthz")) } - certCache := certcache.New(certs, config.OCSPCache) - if err = certCache.Init(nil); err != nil { - die(errors.Wrap(err, "building cert cache")) - } rtvCache, err := rtv.New() if err != nil { die(errors.Wrap(err, "initializing rtv cache")) @@ -134,7 +126,7 @@ func main() { } } - signer, err := signer.New(certs[0], key, config.URLSet, rtvCache, certCache.IsHealthy, + signer, err := signer.New(certCache, key, config.URLSet, rtvCache, certCache.IsHealthy, overrideBaseURL, /*requireHeaders=*/!*flagDevelopment, config.ForwardedRequestHeaders) if err != nil { die(errors.Wrap(err, "building signer")) @@ -151,7 +143,7 @@ func main() { Addr: addr, // Don't use DefaultServeMux, per // https://blog.cloudflare.com/exposing-go-on-the-internet/. - Handler: logIntercept{mux.New(certCache, signer, validityMap)}, + Handler: logIntercept{mux.New(certCache, signer, validityMap, healthz)}, ReadTimeout: 10 * time.Second, ReadHeaderTimeout: 5 * time.Second, // If needing to stream the response, disable WriteTimeout and diff --git a/cmd/amppkg/ocsp.go b/cmd/amppkg/ocsp.go new file mode 100644 index 000000000..0b3d39173 --- /dev/null +++ b/cmd/amppkg/ocsp.go @@ -0,0 +1,49 @@ +package main + +import ( + "crypto" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "time" + + "golang.org/x/crypto/ocsp" +) + +// A fake OCSP responder for the SXG certificate. Abstracted from certcache so as not to give it direct access to the private key. +// Assumes that the given cert is self-signed, and therefore that the +// configured KeyFile also corresponds to the cert's issuer. +type fakeOCSPResponder struct { + key crypto.Signer +} + +// Generates a current OCSP response for the given certificate. +func (this fakeOCSPResponder) Respond(cert *x509.Certificate) ([]byte, error) { + thisUpdate := time.Now() + + // Construct args to ocsp.CreateResponse. + template := ocsp.Response{ + SerialNumber: cert.SerialNumber, + Status: ocsp.Good, + ThisUpdate: thisUpdate, + NextUpdate: thisUpdate.Add(time.Hour * 24 * 7), + IssuerHash: crypto.SHA256, + } + subjectDER, err := asn1.Marshal(pkix.Name{CommonName: "fake-responder.example"}.ToRDNSequence()) + if err != nil { + return nil, err + } + responderCert := x509.Certificate{ + // This is the only field that ocsp.CreateResponse reads. + RawSubject: subjectDER, + } + resp, err := ocsp.CreateResponse(cert, &responderCert, template, this.key) + if err != nil { + return nil, err + } + _, err = ocsp.ParseResponseForCert(resp, cert, cert) + if err != nil { + return nil, err + } + return resp, nil +} diff --git a/cmd/amppkg_dl_sxg/main.go b/cmd/amppkg_dl_sxg/main.go index cdebd2c39..0a849529b 100644 --- a/cmd/amppkg_dl_sxg/main.go +++ b/cmd/amppkg_dl_sxg/main.go @@ -7,7 +7,9 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "os" + "path" "regexp" "strconv" @@ -17,6 +19,7 @@ import ( var flagOutSXG = flag.String("out_sxg", "test.sxg", "Path to where the signed-exchange should be saved.") var flagOutCert = flag.String("out_cert", "test.cert", "Path to where the cert-chain+cbor should be saved.") +var flagCertUrlBase = flag.String("cert_url_base", "", "Override scheme, hostname and parent path in cert-url.") func getSXG(url string) ([]byte, error) { req, err := http.NewRequest("GET", url, nil) @@ -66,6 +69,12 @@ func getCert(url string) ([]byte, error) { if err != nil { return nil, errors.WithStack(err) } + if resp.StatusCode != 200 { + return nil, errors.Errorf("cert-url response error: %s", resp.Status) + } + if contentType := resp.Header.Get("Content-Type"); contentType != "application/cert-chain+cbor" { + return nil, errors.Errorf("invalid content-type of cert-url: %s", contentType) + } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { @@ -90,11 +99,24 @@ func main() { if err != nil { log.Fatalf("%+v", err) } + cURL, err := url.Parse(certURL) + if err != nil { + log.Fatalf("%+v", err) + } + if *flagCertUrlBase != "" { + fURL, err := url.Parse(*flagCertUrlBase) + if err != nil { + log.Fatalf("%+v", err) + } + certHash := path.Base(cURL.Path) + cURL = fURL + cURL.Path = path.Join(cURL.Path, certHash) + } err = ioutil.WriteFile(*flagOutSXG, sxg, 0644) if err != nil { log.Fatalf("%+v", err) } - cert, err := getCert(certURL) + cert, err := getCert(cURL.String()) if err != nil { log.Fatalf("%+v", err) } diff --git a/cmd/gateway_server/server.go b/cmd/gateway_server/server.go index bf9c7355e..7d1340ed2 100644 --- a/cmd/gateway_server/server.go +++ b/cmd/gateway_server/server.go @@ -21,6 +21,7 @@ import ( "github.com/WICG/webpackage/go/signedexchange" "github.com/WICG/webpackage/go/signedexchange/certurl" pb "github.com/ampproject/amppackager/cmd/gateway_server/gateway" + "github.com/ampproject/amppackager/packager/certcache" "github.com/ampproject/amppackager/packager/rtv" "github.com/ampproject/amppackager/packager/signer" "github.com/ampproject/amppackager/packager/util" @@ -38,8 +39,8 @@ type gatewayServer struct { rtvCache *rtv.RTVCache } -func shouldPackage() bool { - return true +func shouldPackage() error { + return nil } func errorToSXGResponse(err error) *pb.SXGResponse { @@ -72,6 +73,9 @@ func (s *gatewayServer) GenerateSXG(ctx context.Context, request *pb.SXGRequest) return errorToSXGResponse(err), nil } + // Note: do not initialize certCache, we just want it to hold the certs for now. + certCache := certcache.New(certs, nil, []string{""}, "", "", "", nil); + privateKey, err := util.ParsePrivateKey(request.PrivateKey) if err != nil { return errorToSXGResponse(err), nil @@ -112,7 +116,7 @@ func (s *gatewayServer) GenerateSXG(ctx context.Context, request *pb.SXGRequest) }, } - packager, err := signer.New(certs[0], privateKey, urlSets, s.rtvCache, shouldPackage, signUrl, false, []string{}) + packager, err := signer.New(certCache, privateKey, urlSets, s.rtvCache, shouldPackage, signUrl, false, []string{}) if err != nil { return errorToSXGResponse(err), nil diff --git a/cmd/transform/transform.go b/cmd/transform/transform.go index 9db9792e0..2e2f7cc03 100644 --- a/cmd/transform/transform.go +++ b/cmd/transform/transform.go @@ -34,7 +34,7 @@ import ( var documentURLFlag = flag.String("url", "", "The URL of the document being processed, e.g. https://example.com/amphtml/article1234") var configFlag = flag.String("config", "DEFAULT", "The configuration that determines the transformations to run. Valid values are DEFAULT, NONE, VALIDATION. See transformer.go for more info.") -var skipNewlineFlag = flag.Bool("noeol", false, "do not output the trailing newline") +var skipNewlineFlag = flag.Bool("noeol", true, "do not output the trailing newline") func checkErr(e error) { if e != nil { diff --git a/docker/Dockerfile b/docker/Dockerfile index d8189404e..95f157327 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ FROM golang:1.11 RUN go get -v github.com/ampproject/amppackager/cmd/amppkg # Seed the ocsp cache -WORKDIR /go/src/github.com/ampproject/amppackager/testdata/b1/ +WORKDIR /go/src/github.com/ampproject/amppackager/testdata/b3/ RUN ./seedcache.sh WORKDIR /go/src/app diff --git a/docker/amppkg.example.toml b/docker/amppkg.example.toml index 31fd5248e..699405ede 100644 --- a/docker/amppkg.example.toml +++ b/docker/amppkg.example.toml @@ -3,8 +3,8 @@ # See https://github.com/ampproject/amppackager/blob/master/amppkg.example.toml # for more information and details on other possible config options. -CertFile = '/go/src/github.com/ampproject/amppackager/testdata/b1/fullchain.cert' -KeyFile = '/go/src/github.com/ampproject/amppackager/testdata/b1/server.privkey' +CertFile = '/go/src/github.com/ampproject/amppackager/testdata/b3/fullchain.cert' +KeyFile = '/go/src/github.com/ampproject/amppackager/testdata/b3/server.privkey' OCSPCache = '/tmp/amppkg-ocsp' [[URLSet]] diff --git a/docs/cache_requirements.md b/docs/cache_requirements.md index 7213d9608..5f122d094 100644 --- a/docs/cache_requirements.md +++ b/docs/cache_requirements.md @@ -20,6 +20,9 @@ These include: * Parameter values of type string, binary, or identifier. * The payload must be: * non-empty. + * well-formed UTF-8 that doesn't contain: + * any characters that cause a parse-error during [HTML preprocessing](https://html.spec.whatwg.org/multipage/parsing.html#preprocessing-the-input-stream) + * U+0000 NULL * valid transformed AMP. The canonical definition of transformed AMP is the return value of [`transform.Process()`](https://github.com/ampproject/amppackager/blob/e4bf0430ba152cfe82ccf063df92021dfc0f26a5/transformer/transformer.go#L219). If given a [valid AMP](https://github.com/ampproject/amphtml/tree/master/validator) @@ -54,6 +57,16 @@ These include: The above is an attempt at a complete list of SXG-related requirements, but it is not guaranteed to be complete. +If a document does not meet all of the above requirements, Google may still use +its payload in an AMP viewer. The requirements for this are approximately as +follows (but should not be relied upon by publishers): + + * magic string is correct + * prologue length fields are correct + * fallback URL matches request URL + * MICE encoding and `Digest` header are valid + * payload is valid AMP + Some of the above limitations are overly strict for an AMP SXG cache's needs, and were implemented as such for the sake of expediency. They may be loosened over time, especially in response to publisher feedback. diff --git a/testdata/b1/index.txt b/fuzz_httpreq/go.mod similarity index 100% rename from testdata/b1/index.txt rename to fuzz_httpreq/go.mod diff --git a/testdata/b1/index.txt.attr b/fuzz_httpresp/go.mod similarity index 100% rename from testdata/b1/index.txt.attr rename to fuzz_httpresp/go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..4e1f85e67 --- /dev/null +++ b/go.mod @@ -0,0 +1,31 @@ +module github.com/ampproject/amppackager + +go 1.13 + +require ( + github.com/WICG/webpackage v0.0.0-20190215052515-70386c3750f2 + github.com/ampproject/amphtml v0.0.0-20180912232012-d3df64d07ae9 + github.com/go-acme/lego/v3 v3.2.0 + github.com/gofrs/flock v0.7.1 + github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.0 + github.com/pelletier/go-toml v1.1.0 + github.com/pkg/errors v0.8.1 + github.com/pquerna/cachecontrol v0.0.0-20180306154005-525d0eb5f91d + github.com/stretchr/testify v1.4.0 + golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 + golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3 + google.golang.org/grpc v1.20.1 + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/square/go-jose.v2 v2.3.1 +) + +replace github.com/davecgh/go-spew => github.com/davecgh/go-spew v1.1.0 + +replace github.com/stretchr/testify => github.com/stretchr/testify v1.2.1 + +replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac + +replace golang.org/x/net => golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24 + +replace golang.org/x/text => golang.org/x/text v0.3.0 diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..69f531285 --- /dev/null +++ b/go.sum @@ -0,0 +1,380 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc= +contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4= +github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= +github.com/Azure/go-autorest/autorest v0.5.0 h1:Mlm9qy2fpQ9MvfyI41G2Zf5B4CsgjjNbLOWszfK6KrY= +github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw= +github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= +github.com/Azure/go-autorest/autorest/adal v0.2.0 h1:7IBDu1jgh+ADHXnEYExkV9RE/ztOOlxdACkkPRthGKw= +github.com/Azure/go-autorest/autorest/adal v0.2.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= +github.com/Azure/go-autorest/autorest/azure/auth v0.1.0 h1:YgO/vSnJEc76NLw2ecIXvXa8bDWiqf1pOJzARAoZsYU= +github.com/Azure/go-autorest/autorest/azure/auth v0.1.0/go.mod h1:Gf7/i2FUpyb/sGBLIFxTBzrNzBo7aPXXE3ZVeDRwdpM= +github.com/Azure/go-autorest/autorest/azure/cli v0.1.0 h1:YTtBrcb6mhA+PoSW8WxFDoIIyjp13XqJeX80ssQtri4= +github.com/Azure/go-autorest/autorest/azure/cli v0.1.0/go.mod h1:Dk8CUAt/b/PzkfeRsWzVG9Yj3ps8mS8ECztu43rdU8U= +github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/mocks v0.1.0 h1:Kx+AUU2Te+A3JIyYn6Dfs+cFgx5XorQKuIXrZGoq/SI= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/to v0.2.0 h1:nQOZzFCudTh+TvquAtCRjM01VEYx85e9qbwt5ncW4L8= +github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/validation v0.1.0 h1:ISSNzGUh+ZSzizJWOWzs8bwpXIePbGLW4z/AmUFGH5A= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.1.0 h1:TRBxC5Pj/fIuh4Qob0ZpkggbfT8RC0SubHbpV3p4/Vc= +github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 h1:xPMsUicZ3iosVPSIP7bW5EcGUzjiiMl1OYTe14y/R24= +github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87/go.mod h1:iGLljf5n9GjT6kc0HBvyI1nOKnGQbNB66VzSNbK5iks= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/WICG/webpackage v0.0.0-20190215052515-70386c3750f2 h1:IJIQQSysq7xk5oO0/6fbYZY0ZLz/KMM+k63ZrJphef8= +github.com/WICG/webpackage v0.0.0-20190215052515-70386c3750f2/go.mod h1:RmCSqjSsrBtTrYiO+OKMBfMgztnTnFXVSapi/mNB7IA= +github.com/akamai/AkamaiOPEN-edgegrid-golang v0.9.0 h1:rXPPPxDA4GCPN0YWwyVHMzcxVpVg8gai2uGhJ3VqOSs= +github.com/akamai/AkamaiOPEN-edgegrid-golang v0.9.0/go.mod h1:zpDJeKyp9ScW4NNrbdr+Eyxvry3ilGPewKoXw3XGN1k= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee h1:NYqDBPkhVYt68W3yoGoRRi32i3MLx2ey7SFkJ1v/UI0= +github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ= +github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/ampproject/amphtml v0.0.0-20180912232012-d3df64d07ae9 h1:0Du3+SEeaZ0yE4G1clQf8npsZUpf2ySFekYfoq2uNZM= +github.com/ampproject/amphtml v0.0.0-20180912232012-d3df64d07ae9/go.mod h1:VdGPUI5OhDH1JDFyOSvCz3x2Ap3YmdLvJ0QboKqoc1c= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/aws/aws-sdk-go v1.23.0 h1:ilfJN/vJtFo1XDFxB2YMBYGeOvGZl6Qow17oyD4+Z9A= +github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4= +github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY= +github.com/cpu/goacmedns v0.0.1 h1:GeIU5chKys9zmHgOAgP+bstRaLqcGQ6HJh/hLw9hrus= +github.com/cpu/goacmedns v0.0.1/go.mod h1:sesf/pNnCYwUevQEQfEwY0Y3DydlQWSGZbaMElOWxok= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decker502/dnspod-go v0.2.0 h1:6dwhUFCYbC5bgpebLKn7PrI43e/5mn9tpUL9YcYCdTU= +github.com/decker502/dnspod-go v0.2.0/go.mod h1:qsurYu1FgxcDwfSwXJdLt4kRsBLZeosEb9uq4Sy+08g= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2 h1:G9/PqfhOrt8JXnw0DGTfVoOkKHDhOlEZqhE/cu+NvQM= +github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnsimple/dnsimple-go v0.30.0 h1:IBIrn9jMKRMwporIRwdFyKdnHXVmwy6obnguB+ZMDIY= +github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/exoscale/egoscale v0.18.1 h1:1FNZVk8jHUx0AvWhOZxLEDNlacTU0chMXUUNkm9EZaI= +github.com/exoscale/egoscale v0.18.1/go.mod h1:Z7OOdzzTOz1Q1PjQXumlz9Wn/CddH0zSYdCF3rnBKXE= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-acme/lego/v3 v3.2.0 h1:z0zvNlL1niv/1qA06V5X1BRC5PeLoGKAlVaWthXQz9c= +github.com/go-acme/lego/v3 v3.2.0/go.mod h1:074uqt+JS6plx+c9Xaiz6+L+GBb+7itGtzfcDM2AhEE= +github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w= +github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/flock v0.7.1 h1:DP+LD/t0njgoPBvT5MJLeliUIVQR03hiKR6vezdwHlc= +github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gophercloud/gophercloud v0.3.0 h1:6sjpKIpVwRIIwmcEGp+WwNovNsem+c+2vm6oxshRpL8= +github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df h1:MZf03xP9WdakyXhOWuAD5uPK3wHh96wCsqe3hCMKh8E= +github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4= +github.com/influxdata/influxdb v1.6.3/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181 h1:TrxPzApUukas24OMMVDUMlCs1XCExJtnGaDEiIAR4oQ= +github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/labbsr0x/bindman-dns-webhook v1.0.2 h1:I7ITbmQPAVwrDdhd6dHKi+MYJTJqPCK0jE6YNBAevnk= +github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= +github.com/labbsr0x/goh v1.0.1 h1:97aBJkDjpyBZGPbQuOK5/gHcSFbcr5aRsq3RSRJFpPk= +github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= +github.com/linode/linodego v0.10.0 h1:AMdb82HVgY8o3mjBXJcUv9B+fnJjfDMn2rNRGbX+jvM= +github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA= +github.com/liquidweb/liquidweb-go v1.6.0 h1:vIj1I/Wf97fUnyirD+bi6Y63c0GiXk9nKI1+sFFl3G0= +github.com/liquidweb/liquidweb-go v1.6.0/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= +github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mrichman/hargo v0.1.2-0.20190117125451-162adce4527e/go.mod h1:ycD51zRGXcO6ak4DnFPjHv4xzbgRU5tYyWDzbMzFYKw= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 h1:o6uBwrhM5C8Ll3MAAxrQxRHEu7FkapwTuI2WmL1rw4g= +github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nrdcg/auroradns v1.0.0 h1:b+NpSqNG6HzMqX2ohGQe4Q/G0WQq8pduWCiZ19vdLY8= +github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/HzqidhOhjw= +github.com/nrdcg/goinwx v0.6.1 h1:AJnjoWPELyCtofhGcmzzcEMFd9YdF2JB/LgutWsWt/s= +github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ= +github.com/nrdcg/namesilo v0.2.1 h1:kLjCjsufdW/IlC+iSfAqj0iQGgKjlbUUeDJio5Y6eMg= +github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/oracle/oci-go-sdk v7.0.0+incompatible h1:oj5ESjXwwkFRdhZSnPlShvLWYdt/IZ65RQxveYM3maA= +github.com/oracle/oci-go-sdk v7.0.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888= +github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014 h1:37VE5TYj2m/FLA9SNr4z0+A0JefvTmR60Zwf8XSEV7c= +github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014/go.mod h1:joRatxRJaZBsY3JAOEMcoOp05CnZzsx4scTxi95DHyQ= +github.com/pelletier/go-toml v1.1.0 h1:cmiOvKzEunMsAxyhXSzpL5Q1CRKpVv0KQsnAIcSEVYM= +github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20180306154005-525d0eb5f91d h1:7gXyC293Lsm2YWgQ+0uaAFFFDO82ruiQSwc3ua+Vtlc= +github.com/pquerna/cachecontrol v0.0.0-20180306154005-525d0eb5f91d/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sacloud/libsacloud v1.26.1 h1:td3Kd7lvpSAxxHEVpnaZ9goHmmhi0D/RfP0Rqqf/kek= +github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/timewasted/linode v0.0.0-20160829202747-37e84520dcf7 h1:CpHxIaZzVy26GqJn8ptRyto8fuoYOd1v0fXm9bG3wQ8= +github.com/timewasted/linode v0.0.0-20160829202747-37e84520dcf7/go.mod h1:imsgLplxEC/etjIhdr3dNzV3JeT27LbVu5pYWm0JCBY= +github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f h1:clyOmELPZd2LuFEyuo1mP6RXpbAW75PwD+RfDj4kBm0= +github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f/go.mod h1:i0f4R4o2HM0m3DZYQWsj6/MEowD57VzoH0v3d7igeFY= +github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo= +github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= +github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4= +github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vultr/govultr v0.1.4 h1:UnNMixYFVO0p80itc8PcweoVENyo1PasfvwKhoasR9U= +github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 h1:d9qaMM+ODpCq+9We41//fu/sHsTnXcrqd1en3x+GKy4= +go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277/go.mod h1:2X8KaoNd1J0lZV+PxJk/5+DGbO/tpwLR1m++a7FnB/Y= +golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac h1:7d7lG9fHOLdL6jZPtnV4LpI41SbohIJ1Atq7U991dMg= +golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24 h1:mEsFm194MmS9vCwxFy+zwu0EU7ZkxxMD1iH++vmGdUY= +golang.org/x/net v0.0.0-20180808004115-f9ce57c11b24/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b h1:ag/x1USPSsqHud38I9BAC88qdNLDHHtQ4mlgQIZPPNA= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3 h1:4y9KwBHBgBNwDbtu44R5o1fdOCQUEXhbk/P4A9WmJq0= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.8.0 h1:VGGbLNyPF7dvYHhcUGYBBGCRDDK0RRJAI6KCvo0CL+E= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/h2non/gock.v1 v1.0.15 h1:SzLqcIlb/fDfg7UvukMpNcWsu7sI5tWwL+KCATZqks0= +gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0= +gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc h1:GAcf+t0o8gdJAdSFYdE9wChu4bIyguMVqz0RHiFL5VY= +gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw= +gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/packager/certcache/certcache.go b/packager/certcache/certcache.go index 96f45e85b..80bfd6a23 100644 --- a/packager/certcache/certcache.go +++ b/packager/certcache/certcache.go @@ -17,6 +17,7 @@ package certcache import ( "bytes" "context" + "crypto" "crypto/x509" "encoding/base64" "io" @@ -29,6 +30,8 @@ import ( "time" "github.com/WICG/webpackage/go/signedexchange/certurl" + "github.com/ampproject/amppackager/packager/certfetcher" + "github.com/ampproject/amppackager/packager/certloader" "github.com/ampproject/amppackager/packager/mux" "github.com/ampproject/amppackager/packager/util" "github.com/pkg/errors" @@ -51,15 +54,55 @@ const maxOCSPResponseBytes = 1024 * 1024 // How often to check if OCSP stapling needs updating. const ocspCheckInterval = 1 * time.Hour +// How often to check if certs needs updating. +const certCheckInterval = 24 * time.Hour + +// Max number of OCSP request tries. +// This will timeout after 1 + 2 + 4 + 8 + 10 * 6 = 75 minutes. +const maxOCSPTries = 10 + +// Recommended renewal duration for certs. This is duration before next cert expiry. +// 8 days is recommended duration to start requesting new certs to allow for ACME server outages. +// It's 6 days + 2 days renewal grace period. +// 6 days so that generated SXGs are valid for their full lifetime, plus 2 days in front of that to allow time for the new cert +// to be obtained. +// TODO(banaag): make 2 days renewal grace period configurable in toml. +const certRenewalInterval = 8 * 24 * time.Hour + +type OCSPResponder func(*x509.Certificate) ([]byte, error) + +type CertHandler interface { + GetLatestCert() *x509.Certificate + IsHealthy() error +} + type CertCache struct { // TODO(twifkak): Support multiple cert chains (for different domains, for different roots). - certName string - certs []*x509.Certificate + certName string + certsMu sync.RWMutex + certs []*x509.Certificate + // If certFetcher is not set, that means cert auto-renewal is not available. + certFetcher *certfetcher.CertFetcher + renewedCertsMu sync.RWMutex + renewedCertName string + renewedCerts []*x509.Certificate ocspUpdateAfterMu sync.RWMutex ocspUpdateAfter time.Time + stop chan struct{} // TODO(twifkak): Implement a registry of Updateable instances which can be configured in the toml. - ocspFile Updateable - client http.Client + ocspFile Updateable + ocspFilePath string + client http.Client + // Given a certificate, returns a current OCSP response for the cert; + // this is a fallback, called when in development mode and there is no + // OCSP URL. + generateOCSPResponse OCSPResponder + // Domains to validate + Domains []string + CertFile string + NewCertFile string + // Is CertCache initialized to do cert renewal or OCSP refreshes? + isInitialized bool // "Virtual methods", exposed for testing. // Given a certificate, returns the OCSP responder URL for that cert. @@ -68,11 +111,25 @@ type CertCache struct { httpExpiry func(*http.Request, *http.Response) time.Time } -// Must call Init() on the returned CertCache before you can use it. -func New(certs []*x509.Certificate, ocspCache string) *CertCache { +// Callers need to call Init() on the returned CertCache before the cache can auto-renew certs. +// Callers can use the uninitialized CertCache for testing certificates (without doing OCSP or +// cert refreshes). +// +// TODO(banaag): per gregable@ comments: +// The long argument list makes the callsites tricky to read and easy to get wrong, especially if several of the arguments have the same type. +// +// An alternative pattern would be to create an IsInitialized() bool or similarly named function that verifies all of the required fields have +// been set. Then callers can just set fields in the struct by name and assert IsInitialized before doing anything with it. +func New(certs []*x509.Certificate, certFetcher *certfetcher.CertFetcher, domains []string, + certFile string, newCertFile string, ocspCache string, generateOCSPResponse OCSPResponder) *CertCache { + certName := "" + if len(certs) > 0 && certs[0] != nil { + certName = util.CertName(certs[0]) + } return &CertCache{ - certName: util.CertName(certs[0]), + certName: certName, certs: certs, + certFetcher: certFetcher, ocspUpdateAfter: infiniteFuture, // Default, in case initial readOCSP successfully loads from disk. // Distributed OCSP cache to support the following sleevi requirements: // 1. Support for keeping a long-lived (disk) cache of OCSP responses. @@ -83,10 +140,13 @@ func New(certs []*x509.Certificate, ocspCache string) *CertCache { // certificate, all needing to staple an OCSP response. You don't // want to have all of them hammering the OCSP server - ideally, // you'd have one request, in the backend, and updating them all. - ocspFile: &Chained{first: &InMemory{}, second: &LocalFile{path: ocspCache}}, - client: http.Client{Timeout: 60 * time.Second}, + ocspFile: &Chained{first: &InMemory{}, second: &LocalFile{path: ocspCache}}, + ocspFilePath: ocspCache, + stop: make(chan struct{}), + generateOCSPResponse: generateOCSPResponse, + client: http.Client{Timeout: 60 * time.Second}, extractOCSPServer: func(cert *x509.Certificate) (string, error) { - if len(cert.OCSPServer) < 1 { + if cert == nil || len(cert.OCSPServer) < 1 { return "", errors.New("Cert missing OCSPServer.") } // This is a URI, per https://tools.ietf.org/html/rfc5280#section-4.2.2.1. @@ -100,12 +160,18 @@ func New(certs []*x509.Certificate, ocspCache string) *CertCache { return expiry } }, + Domains: domains, + CertFile: certFile, + NewCertFile: newCertFile, + isInitialized: false, } } -func (this *CertCache) Init(stop chan struct{}) error { +func (this *CertCache) Init() error { + this.updateCertIfNecessary() + // Prime the OCSP disk and memory cache, so we can start serving immediately. - _, _, err := this.readOCSP() + _, _, err := this.readOCSP(true) if err != nil { return errors.Wrap(err, "initializing CertCache") } @@ -117,11 +183,68 @@ func (this *CertCache) Init(stop chan struct{}) error { // like the OCSP responder giving you junk, but also sufficient time // to raise an alert if something has gone really wrong. // 7. The ability to serve old responses while fetching new responses. - go this.maintainOCSP(stop) + go this.maintainOCSP() + + if this.certFetcher != nil { + // Update Certs in the background. + go this.maintainCerts() + } + + this.isInitialized = true + + return nil +} + +// Stop stops the goroutines spawned in Init, which are automatically updating the certificate and the OCSP response. +// It returns true if the call actually stops them, false if they have already been stopped. +func (this *CertCache) Stop() bool { + select { + // this.stop will never be used for sending a value. Thus this case matches only when it has already been closed. + case <-this.stop: + return false + default: + close(this.stop) + return true + } +} + +// Gets the latest cert. +// Returns the current cert if the cache has not been initialized or if the certFetcher is not set (good for testing) +// If cert is invalid, it will attempt to renew. +// If cert is still valid, returns the current cert. +func (this *CertCache) GetLatestCert() *x509.Certificate { + if !this.isInitialized || this.certFetcher == nil { + // If certcache is not initialized or certFetcher is not set, + // just return cert without checking if it needs auto-renewal. + return this.getCert() + } + + if !this.hasCert() { + return nil + } + + d, err := util.GetDurationToExpiry(this.getCert(), time.Now()) + if err != nil { + // Current cert is already invalid. Check if renewal is available. + log.Println("Current cert is expired, attempting to renew: ", err) + this.updateCertIfNecessary() + return this.getCert() + } + if d >= time.Duration(certRenewalInterval) { + // Cert is still valid. + return this.getCert() + } else if d < time.Duration(certRenewalInterval) { + // Cert is still valid, but we need to start process of requesting new cert. + log.Println("Current cert is close to expiry threshold, attempting to renew in the background.") + return this.getCert() + } return nil } func (this *CertCache) createCertChainCBOR(ocsp []byte) ([]byte, error) { + this.certsMu.RLock() + defer this.certsMu.RUnlock() + certChain := make(certurl.CertChain, len(this.certs)) for i, cert := range this.certs { certChain[i] = &certurl.CertChainItem{Cert: cert} @@ -137,7 +260,7 @@ func (this *CertCache) createCertChainCBOR(ocsp []byte) ([]byte, error) { } func (this *CertCache) ocspMidpoint(bytes []byte, issuer *x509.Certificate) (time.Time, error) { - resp, err := ocsp.ParseResponseForCert(bytes, this.certs[0], issuer) + resp, err := ocsp.ParseResponseForCert(bytes, this.getCert(), issuer) if err != nil { return time.Time{}, errors.Wrap(err, "Parsing OCSP") } @@ -146,6 +269,10 @@ func (this *CertCache) ocspMidpoint(bytes []byte, issuer *x509.Certificate) (tim func (this *CertCache) ServeHTTP(resp http.ResponseWriter, req *http.Request) { params := mux.Params(req) + + // RLock for the certName + this.certsMu.RLock() + defer this.certsMu.RUnlock() if params["certName"] == this.certName { // https://tools.ietf.org/html/draft-yasskin-httpbis-origin-signed-exchanges-impl-00#section-3.3 // This content-type is not standard, but included to reduce @@ -153,7 +280,7 @@ func (this *CertCache) ServeHTTP(resp http.ResponseWriter, req *http.Request) { resp.Header().Set("Content-Type", "application/cert-chain+cbor") // Instruct the intermediary to reload this cert-chain at the // OCSP midpoint, in case it cannot parse it. - ocsp, _, err := this.readOCSP() + ocsp, _, err := this.readOCSP(false) if err != nil { util.NewHTTPError(http.StatusInternalServerError, "Error reading OCSP: ", err).LogAndRespond(resp) return @@ -189,47 +316,97 @@ func (this *CertCache) ServeHTTP(resp http.ResponseWriter, req *http.Request) { // 8. Some idea of what to do when "things go bad". // What happens when it's been 7 days, no new OCSP response can be obtained, // and the current response is about to expire? -func (this *CertCache) IsHealthy() bool { - ocsp, _, err := this.readOCSP() - return err != nil || this.isHealthy(ocsp) +func (this *CertCache) IsHealthy() error { + ocsp, _, errorOCSP := this.readOCSP(false) + if errorOCSP != nil { + return errorOCSP + } + errorHealth := this.isHealthy(ocsp) + if errorHealth != nil { + return errorHealth + } + return nil } -func (this *CertCache) isHealthy(ocspResp []byte) bool { +func (this *CertCache) isHealthy(ocspResp []byte) error { if ocspResp == nil { - log.Println("OCSP response not yet fetched.") - return false + return errors.New("OCSP response not yet fetched.") } issuer := this.findIssuer() if issuer == nil { - log.Println("Cannot find issuer certificate in CertFile.") - return false + return errors.New("Cannot find issuer certificate in CertFile.") } - resp, err := ocsp.ParseResponseForCert(ocspResp, this.certs[0], issuer) + resp, err := ocsp.ParseResponseForCert(ocspResp, this.getCert(), issuer) if err != nil { - log.Println("Error parsing OCSP response:", err) - return false + return errors.Wrap(err, "Error parsing OCSP response") } if resp.NextUpdate.Before(time.Now()) { - log.Println("Cached OCSP is stale, NextUpdate:", resp.NextUpdate) - return false + return errors.Errorf("Cached OCSP is stale, NextUpdate: %v", resp.NextUpdate) } - return true + return nil } -// Returns the OCSP response and expiry, refreshing if necessary. -func (this *CertCache) readOCSP() ([]byte, time.Time, error) { +func (this *CertCache) readOCSPHelper(numTries int, exhaustedRetries bool) ([]byte, time.Time, error) { var ocspUpdateAfter time.Time + + this.certsMu.RLock() + defer this.certsMu.RUnlock() ocsp, err := this.ocspFile.Read(context.Background(), this.shouldUpdateOCSP, func(orig []byte) []byte { - return this.fetchOCSP(orig, &ocspUpdateAfter) + return this.fetchOCSP(orig, this.certs, &ocspUpdateAfter, numTries > 0) }) if err != nil { - return nil, time.Time{}, errors.Wrap(err, "Updating OCSP cache") + if exhaustedRetries { + return nil, time.Time{}, errors.Wrap(err, "Updating OCSP cache") + } else { + return nil, time.Time{}, nil + } } if len(ocsp) == 0 { - return nil, time.Time{}, errors.New("Missing OCSP response.") + if exhaustedRetries { + return nil, time.Time{}, errors.New("Missing OCSP response.") + } else { + return nil, time.Time{}, nil + } + } + if err := this.isHealthy(ocsp); err != nil { + if exhaustedRetries { + return nil, time.Time{}, errors.Wrap(err, "OCSP failed health check") + } else { + return nil, time.Time{}, nil + } + } + + return ocsp, ocspUpdateAfter, nil +} + +// Returns the OCSP response and expiry, refreshing if necessary. +func (this *CertCache) readOCSP(allowRetries bool) ([]byte, time.Time, error) { + var ocspUpdateAfter time.Time + var err error + var maxTries int + + ocsp := []byte(nil) + waitTimeInMinutes := 1 + if !allowRetries || this.certFetcher == nil { + // If certFetcher is nil, that means we are not auto-renewing so don't retry OCSP. + maxTries = 1 + } else { + maxTries = maxOCSPTries } - if !this.isHealthy(ocsp) { - return nil, time.Time{}, errors.New("OCSP failed health check.") + + for numTries := 0; numTries < maxTries; { + ocsp, ocspUpdateAfter, err = this.readOCSPHelper(numTries, numTries >= maxTries - 1) + if err != nil { + return nil, ocspUpdateAfter, err + } + if !this.shouldUpdateOCSP(ocsp) { + break; + } + // Wait only if are not on our last try. + if numTries < maxTries - 1 { + waitTimeInMinutes = waitForSpecifiedTime(waitTimeInMinutes, numTries) + } + numTries++ } this.ocspUpdateAfterMu.Lock() defer this.ocspUpdateAfterMu.Unlock() @@ -239,10 +416,28 @@ func (this *CertCache) readOCSP() ([]byte, time.Time, error) { this.ocspUpdateAfter = ocspUpdateAfter } return ocsp, ocspUpdateAfter, nil + +} + +// Print # of retries, wait for specified time and returned updated wait time. +func waitForSpecifiedTime(waitTimeInMinutes int, numRetries int) int { + log.Printf("Retrying OCSP server: retry #%d\n", numRetries) + // Wait using exponential backoff. + log.Printf("Waiting for %d minute(s)\n", waitTimeInMinutes) + waitTimeDuration := time.Duration(waitTimeInMinutes) * time.Minute + // For exponential backoff. + newWaitTimeInMinutes := 2 * waitTimeInMinutes + if newWaitTimeInMinutes > 10 { + // Cap the wait time at 10 minutes. + newWaitTimeInMinutes = 10 + } + time.Sleep(waitTimeDuration) + return newWaitTimeInMinutes } -// Checks for OCSP updates every hour. Never terminates. -func (this *CertCache) maintainOCSP(stop chan struct{}) { +// Checks for OCSP updates every hour. Terminates only when stop receives +// a message. +func (this *CertCache) maintainOCSP() { // Only make one request per ocspCheckInterval, to minimize the impact // on OCSP servers that are buckling under load, per sleevi requirement: // 5. As with any system doing background requests on a remote server, @@ -256,11 +451,11 @@ func (this *CertCache) maintainOCSP(stop chan struct{}) { for { select { case <-ticker.C: - _, _, err := this.readOCSP() + _, _, err := this.readOCSP(true) if err != nil { log.Println("Warning: OCSP update failed. Cached response may expire:", err) } - case <-stop: + case <-this.stop: ticker.Stop() return } @@ -268,8 +463,8 @@ func (this *CertCache) maintainOCSP(stop chan struct{}) { } // Returns true if OCSP is expired (or near enough). -func (this *CertCache) shouldUpdateOCSP(bytes []byte) bool { - if len(bytes) == 0 { +func (this *CertCache) shouldUpdateOCSP(ocsp []byte) bool { + if len(ocsp) == 0 { // TODO(twifkak): Use a logging framework with support for debug-only statements. log.Println("Updating OCSP; none cached yet.") return true @@ -281,7 +476,7 @@ func (this *CertCache) shouldUpdateOCSP(bytes []byte) bool { return false } // Compute the midpoint per sleevi #3 (see above). - midpoint, err := this.ocspMidpoint(bytes, issuer) + midpoint, err := this.ocspMidpoint(ocsp, issuer) if err != nil { log.Println("Error computing OCSP midpoint:", err) return true @@ -311,8 +506,22 @@ func (this *CertCache) shouldUpdateOCSP(bytes []byte) bool { // Finds the issuer of this cert (i.e. the second from the bottom of the // chain). func (this *CertCache) findIssuer() *x509.Certificate { - issuerName := this.certs[0].Issuer - for _, cert := range this.certs { + if !this.hasCert() { + return nil + } + return this.findIssuerUsingCerts(this.certs) +} + +// Finds the issuer of the specified cert (i.e. the second from the bottom of the +// chain). +func (this *CertCache) findIssuerUsingCerts(certs []*x509.Certificate) *x509.Certificate { + if certs == nil || len(certs) == 0 { + return nil + } + this.certsMu.RLock() + defer this.certsMu.RUnlock() + issuerName := certs[0].Issuer + for _, cert := range certs { // The subject name is guaranteed to match the issuer name per // https://tools.ietf.org/html/rfc3280#section-4.1.2.4 and // #section-4.1.2.6. (The latter guarantees that the subject @@ -334,25 +543,33 @@ func (this *CertCache) findIssuer() *x509.Certificate { } // Queries the OCSP responder for this cert and return the OCSP response. -func (this *CertCache) fetchOCSP(orig []byte, ocspUpdateAfter *time.Time) []byte { - issuer := this.findIssuer() +func (this *CertCache) fetchOCSP(orig []byte, certs []*x509.Certificate, ocspUpdateAfter *time.Time, isRetry bool) []byte { + issuer := this.findIssuerUsingCerts(certs) if issuer == nil { log.Println("Cannot find issuer certificate in CertFile.") return orig } - // The default SHA1 hash function is mandated by the Lightweight OCSP // Profile, https://tools.ietf.org/html/rfc5019 2.1.1 (sleevi #4, see above). - req, err := ocsp.CreateRequest(this.certs[0], issuer, nil) + req, err := ocsp.CreateRequest(certs[0], issuer, nil) if err != nil { log.Println("Error creating OCSP request:", err) return orig } - ocspServer, err := this.extractOCSPServer(this.certs[0]) + ocspServer, err := this.extractOCSPServer(certs[0]) if err != nil { - log.Println("Error extracting OCSP server:", err) - return orig + if this.generateOCSPResponse == nil { + log.Println("Error extracting OCSP server:", err) + return orig + } + log.Println("Cert lacks OCSP URL; using fake OCSP in development mode.") + resp, err := this.generateOCSPResponse(certs[0]) + if err != nil { + log.Println("error generating fake OCSP response:", err) + return orig + } + return resp } // Conform to the Lightweight OCSP Profile, by preferring GET over POST @@ -364,7 +581,8 @@ func (this *CertCache) fetchOCSP(orig []byte, ocspUpdateAfter *time.Time) []byte // StdEncoding). getURL := ocspServer + "/" + url.PathEscape(base64.StdEncoding.EncodeToString(req)) var httpReq *http.Request - if len(getURL) <= 255 { + // Logic is a fallback, due to some CAs not responding as expected to a GET. + if len(getURL) <= 255 && !isRetry { httpReq, err = http.NewRequest("GET", getURL, nil) if err != nil { log.Println("Error creating OCSP response:", err) @@ -393,16 +611,17 @@ func (this *CertCache) fetchOCSP(orig []byte, ocspUpdateAfter *time.Time) []byte // expiry earlier than we'd usually follow. *ocspUpdateAfter = this.httpExpiry(httpReq, httpResp) - respBytes, err := ioutil.ReadAll(io.LimitReader(httpResp.Body, 1024*1024)) + respBytes, err := ioutil.ReadAll(io.LimitReader(httpResp.Body, maxOCSPResponseBytes)) if err != nil { log.Println("Error reading OCSP response:", err) return orig } + // Validate the response, per sleevi requirement: // 2. Validate the server responses to make sure it is something the client will accept. // and also per sleevi #4 (see above), as required by // https://tools.ietf.org/html/rfc5019#section-2.2.2. - resp, err := ocsp.ParseResponseForCert(respBytes, this.certs[0], issuer) + resp, err := ocsp.ParseResponseForCert(respBytes, certs[0], issuer) if err != nil { log.Println("Error parsing OCSP response:", err) return orig @@ -422,9 +641,248 @@ func (this *CertCache) fetchOCSP(orig []byte, ocspUpdateAfter *time.Time) []byte // OCSP duration must be <=7 days, per // https://wicg.github.io/webpackage/draft-yasskin-httpbis-origin-signed-exchanges-impl.html#cross-origin-trust. // Serving these responses may cause UAs to reject the SXG. - if resp.NextUpdate.Sub(resp.ThisUpdate) > time.Hour * 24 * 7 { + if resp.NextUpdate.Sub(resp.ThisUpdate) > time.Hour*24*7 { log.Printf("OCSP nextUpdate %+v too far ahead of thisUpdate %+v\n", resp.NextUpdate, resp.ThisUpdate) return orig } return respBytes } + +// Checks for cert updates every certCheckInterval hours. Terminates only when stop +// receives a message. +func (this *CertCache) maintainCerts() { + // Only make one request per certCheckInterval, to minimize the impact + // on servers that are buckling under load. + ticker := time.NewTicker(certCheckInterval) + + for { + select { + case <-ticker.C: + this.updateCertIfNecessary() + case <-this.stop: + ticker.Stop() + return + } + } +} + +// Returns true iff cert cache contains at least 1 cert. +func (this *CertCache) hasCert() bool { + this.certsMu.RLock() + defer this.certsMu.RUnlock() + return len(this.certs) > 0 && this.certs[0] != nil +} + +func (this *CertCache) getCert() *x509.Certificate { + if !this.hasCert() { + return nil + } + this.certsMu.RLock() + defer this.certsMu.RUnlock() + return this.certs[0] +} + +// Returns true iff cert cache renewal contains at least 1 cert. +func (this *CertCache) hasRenewalCert() bool { + this.renewedCertsMu.RLock() + defer this.renewedCertsMu.RUnlock() + return len(this.renewedCerts) > 0 && this.renewedCerts[0] != nil +} + +func (this *CertCache) getRenewalCert() *x509.Certificate { + this.renewedCertsMu.RLock() + defer this.renewedCertsMu.RUnlock() + if !this.hasRenewalCert() { + return nil + } + return this.renewedCerts[0] +} + +// Set current cert with mutex protection. +func (this *CertCache) setCerts(certs []*x509.Certificate) { + this.certsMu.Lock() + defer this.certsMu.Unlock() + this.certs = certs + this.certName = util.CertName(certs[0]) + + err := certloader.WriteCertsToFile(this.certs, this.CertFile) + if err != nil { + log.Printf("Unable to write certs to file: %s", this.CertFile) + } + + // Purge OCSP cache + certloader.RemoveFile(this.ocspFilePath) +} + +// Set new cert with mutex protection. +func (this *CertCache) setNewCerts(certs []*x509.Certificate) { + this.renewedCertsMu.Lock() + defer this.renewedCertsMu.Unlock() + this.renewedCerts = certs + + if this.renewedCerts == nil { + this.renewedCertName = "" + err := certloader.RemoveFile(this.NewCertFile) + if err != nil { + log.Printf("Unable to remove file: %s", this.NewCertFile) + } + return + } + this.renewedCertName = util.CertName(certs[0]) + + err := certloader.WriteCertsToFile(this.renewedCerts, this.NewCertFile) + if err != nil { + log.Printf("Unable to write certs to file: %s", this.NewCertFile) + } +} + +// Update the cert in the cache if necessary. +func (this *CertCache) updateCertIfNecessary() { + log.Println("Updating cert if necessary") + if this.certFetcher == nil { + // Don't request new certs from CA if certFetcher is not set. This means this instance of the amppackager + // is not in autorenewcert mode. Just make an attempt at reading the cert saved on disk to see if + // another amppackager instance that is in autorenewcert mode actually updated it with a valid cert. + log.Println("Certfetcher is not set, skipping cert updates. Checking cert on disk if updated.") + this.reloadCertIfExpired() + return + } + d := time.Duration(0) + err := errors.New("") + if this.hasCert() { + d, err = util.GetDurationToExpiry(this.getCert(), time.Now()) + } + if err != nil { + this.renewedCertsMu.Lock() + defer this.renewedCertsMu.Unlock() + + // Current cert is already invalid, check if we have a pending renewal cert. + if this.renewedCerts != nil { + // If renewedCerts is set, copy that over to certs + // and set renewedCerts to nil. + this.setCerts(this.renewedCerts) + this.setNewCerts(nil) + return + } + // Current cert is already invalid. Try refreshing. + log.Println("Warning current cert is expired, attempting to renew: ", err) + certs, err := this.certFetcher.FetchNewCert() + if err != nil { + log.Println("Error trying to fetch new certificates from CA: ", err) + return + } + this.setCerts(certs) + return + } + if d >= time.Duration(certRenewalInterval) { + // Cert is still valid, don't do anything. + } else if d < time.Duration(certRenewalInterval) { + this.renewedCertsMu.Lock() + defer this.renewedCertsMu.Unlock() + + // Check if we already have a renewal cert waiting, fetch a new cert if not. + if this.renewedCerts == nil { + // Cert is still valid, but we need to start process of requesting new cert. + log.Println("Warning: Current cert crossed threshold for renewal, attempting to renew.") + certs, err := this.certFetcher.FetchNewCert() + if err != nil { + log.Println("Error trying to fetch new certificates from CA: ", err) + return + } + this.setNewCerts(certs) + } else { + // TODO(banaag) from twifkak comments: + // Note that this logic works, albeit might fail to fetch OCSP the first try, but will succeed 24 hours later. + // + // I realize it's difficult to use readOCSP here, since it's hard-coded to use this.certs and friends, rather than + // this.renewedCerts and friends. That makes me think two things: + // + // The extraction of the retry logic from readOCSP would be useful. + // We should bundle certName, certs, certsMu, ocspFile, ocspFilePath, ocspUpdateAfter, and ocspUpdateAfterMu into + // a new struct type, and then have two copies of that in certcache - one for current certs and one for new certs. + var ocspUpdateAfter time.Time + + ocsp, _, errorOCSP := this.readOCSP(true) + if errorOCSP != nil { + newOCSP := this.fetchOCSP(ocsp, this.renewedCerts, &ocspUpdateAfter, false) + // Check if newOCSP != ocsp and that there are no errors, health-wise with new ocsp. + if !bytes.Equal(newOCSP, ocsp) && this.isHealthy(newOCSP) == nil { + // We were able to fetch new OCSP with renewal cert, time to switch to new certs. + this.setCerts(this.renewedCerts) + this.setNewCerts(nil) + } + } + } + } +} + +func (this *CertCache) doesCertNeedReloading() bool { + if !this.hasCert() { return true } + d, err := util.GetDurationToExpiry(this.getCert(), time.Now()) + return err != nil || d < certRenewalInterval +} + +func (this *CertCache) reloadCertIfExpired() { + if !this.doesCertNeedReloading() { + return + } + + // If we get to here, the cert was either expired, or it's time to renew. + // We always validate the certs here. If we are in development mode and the certs don't validate, + // it doesn't matter because the old certs won't be overridden (and the old certs are probably invalid, too). + certs, err := certloader.LoadAndValidateCertsFromFile(this.CertFile, true) + if err != nil { + log.Println(errors.Wrap(err, "Can't load cert file")) + certs = nil + } + if certs != nil { + this.setCerts(certs) + } + + newCerts, err := certloader.LoadAndValidateCertsFromFile(this.NewCertFile, true) + if err != nil { + log.Println(errors.Wrap(err, "Can't load new cert file")) + newCerts = nil + } + if newCerts != nil { + this.setNewCerts(newCerts) + } +} + +// Creates cert cache by loading certs and keys from disk, doing validation +// and populating the cert cache with current set of certificate related information. +// If development mode is true, prints a warning for certs that can't sign HTTP exchanges. +func PopulateCertCache(config *util.Config, key crypto.PrivateKey, generateOCSPResponse OCSPResponder, + developmentMode bool, autoRenewCert bool) (*CertCache, error) { + + if config.CertFile == "" { + return nil, errors.New("Missing cert file path in config.") + } + + if autoRenewCert && config.NewCertFile == "" { + return nil, errors.New("Missing new cert file path in config.") + } + + certs, err := certloader.LoadCertsFromFile(config, developmentMode) + if err != nil { + log.Println(errors.Wrap(err, "Can't load cert file")) + certs = nil + } + domain := "" + for _, urlSet := range config.URLSet { + domain = urlSet.Sign.Domain + if certs != nil { + if err := util.CertificateMatches(certs[0], key, domain); err != nil { + return nil, errors.Wrapf(err, "checking %s", config.CertFile) + } + } + } + + certFetcher, err := certloader.CreateCertFetcher(config, key, domain, developmentMode, autoRenewCert) + if err != nil { + return nil, errors.Wrap(err, "creating cert fetcher from config") + } + certCache := New(certs, certFetcher, []string{domain}, config.CertFile, config.NewCertFile, config.OCSPCache, generateOCSPResponse) + + return certCache, nil +} diff --git a/packager/certcache/certcache_test.go b/packager/certcache/certcache_test.go index 6d84cf988..b8ca00ab0 100644 --- a/packager/certcache/certcache_test.go +++ b/packager/certcache/certcache_test.go @@ -37,13 +37,13 @@ import ( ) var caCert = func() *x509.Certificate { - certPem, _ := ioutil.ReadFile("../../testdata/b1/ca.cert") + certPem, _ := ioutil.ReadFile("../../testdata/b3/ca.cert") certs, _ := signedexchange.ParseCertificates(certPem) return certs[0] }() var caKey = func() *rsa.PrivateKey { - keyPem, _ := ioutil.ReadFile("../../testdata/b1/ca.privkey") + keyPem, _ := ioutil.ReadFile("../../testdata/b3/ca.privkey") key, _ := util.ParsePrivateKey(keyPem) return key.(*rsa.PrivateKey) }() @@ -51,7 +51,7 @@ var caKey = func() *rsa.PrivateKey { func FakeOCSPResponse(thisUpdate time.Time) ([]byte, error) { template := ocsp.Response{ Status: ocsp.Good, - SerialNumber: pkgt.Certs[0].SerialNumber, + SerialNumber: pkgt.B3Certs[0].SerialNumber, ThisUpdate: thisUpdate, NextUpdate: thisUpdate.Add(7 * 24 * time.Hour), RevokedAt: thisUpdate.AddDate( /*years=*/ 0 /*months=*/, 0 /*days=*/, 365), @@ -68,13 +68,19 @@ type CertCacheSuite struct { ocspServerWasCalled bool ocspHandler func(w http.ResponseWriter, req *http.Request) tempDir string - stop chan struct{} handler *CertCache } +func stringPtr(s string) *string { + return &s +} + func (this *CertCacheSuite) New() (*CertCache, error) { // TODO(twifkak): Stop the old CertCache's goroutine. - certCache := New(pkgt.Certs, filepath.Join(this.tempDir, "ocsp")) + // TODO(banaag): Consider adding a test with certfetcher set. + // For now, this tests certcache without worrying about certfetcher. + certCache := New(pkgt.B3Certs, nil, []string{"example.com"}, "cert.crt", "newcert.crt", + filepath.Join(this.tempDir, "ocsp"), nil) certCache.extractOCSPServer = func(*x509.Certificate) (string, error) { return this.ocspServer.URL, nil } @@ -86,7 +92,7 @@ func (this *CertCacheSuite) New() (*CertCache, error) { return defaultHttpExpiry(req, resp) } } - err := certCache.Init(this.stop) + err := certCache.Init() return certCache, err } @@ -114,8 +120,6 @@ func (this *CertCacheSuite) SetupTest() { this.tempDir, err = ioutil.TempDir(os.TempDir(), "certcache_test") this.Require().NoError(err, "setting up test harness") - this.stop = make(chan struct{}) - this.handler, err = this.New() this.Require().NoError(err, "instantiating CertCache") } @@ -125,7 +129,7 @@ func (this *CertCacheSuite) TearDownTest() { this.fakeOCSPExpiry = nil // Reverse SetupTest. - this.stop <- struct{}{} + this.handler.Stop() err := os.RemoveAll(this.tempDir) if err != nil { @@ -134,7 +138,7 @@ func (this *CertCacheSuite) TearDownTest() { } func (this *CertCacheSuite) mux() http.Handler { - return mux.New(this.handler, nil, nil) + return mux.New(this.handler, nil, nil, nil) } func (this *CertCacheSuite) ocspServerCalled(f func()) bool { @@ -181,6 +185,34 @@ func (this *CertCacheSuite) TestServesCertificate() { this.Assert().NotContains(cbor, "sct") } +func (this *CertCacheSuite) TestCertCacheIsHealthy() { + this.Assert().NoError(this.handler.IsHealthy()) +} + +func (this *CertCacheSuite) TestCertCacheIsNotHealthy() { + // Prime memory cache with a past-midpoint OCSP: + err := os.Remove(filepath.Join(this.tempDir, "ocsp")) + this.Require().NoError(err, "deleting OCSP tempfile") + this.fakeOCSP, err = FakeOCSPResponse(time.Now().Add(-4 * 24 * time.Hour)) + this.Require().NoError(err, "creating stale OCSP response") + this.Require().True(this.ocspServerCalled(func() { + this.handler, err = this.New() + this.Require().NoError(err, "reinstantiating CertCache") + })) + + // Prime disk cache with a bad OCSP: + freshOCSP := []byte("0xdeadbeef") + this.fakeOCSP = freshOCSP + err = ioutil.WriteFile(filepath.Join(this.tempDir, "ocsp"), freshOCSP, 0644) + this.Require().NoError(err, "writing fresh OCSP response to disk") + + this.Assert().True(this.ocspServerCalled(func() { + this.handler.readOCSP(true) + })) + + this.Assert().Error(this.handler.IsHealthy()) +} + func (this *CertCacheSuite) TestServes404OnMissingCertificate() { resp := pkgt.Get(this.T(), this.mux(), "/amppkg/cert/lalala") this.Assert().Equal(http.StatusNotFound, resp.StatusCode, "incorrect status: %#v", resp) @@ -203,7 +235,7 @@ func (this *CertCacheSuite) TestOCSP() { func (this *CertCacheSuite) TestOCSPCached() { // Verify it is in the memory cache: this.Assert().False(this.ocspServerCalled(func() { - _, _, err := this.handler.readOCSP() + _, _, err := this.handler.readOCSP(true) this.Assert().NoError(err) })) @@ -231,7 +263,7 @@ func (this *CertCacheSuite) TestOCSPExpiry() { // On update, verify network is called: this.Assert().True(this.ocspServerCalled(func() { - _, _, err := this.handler.readOCSP() + _, _, err := this.handler.readOCSP(true) this.Assert().NoError(err) })) } @@ -255,7 +287,7 @@ func (this *CertCacheSuite) TestOCSPUpdateFromDisk() { // On update, verify network is not called (fresh OCSP from disk is used): this.Assert().False(this.ocspServerCalled(func() { - _, _, err := this.handler.readOCSP() + _, _, err := this.handler.readOCSP(true) this.Assert().NoError(err) })) } @@ -274,7 +306,7 @@ func (this *CertCacheSuite) TestOCSPExpiredViaHTTPHeaders() { // Verify that, 2 seconds later, a new fetch is attempted. this.Assert().True(this.ocspServerCalled(func() { - _, _, err := this.handler.readOCSP() + _, _, err := this.handler.readOCSP(true) this.Require().NoError(err, "updating OCSP") })) } @@ -295,16 +327,41 @@ func (this *CertCacheSuite) TestOCSPIgnoreInvalidUpdate() { this.fakeOCSP, err = FakeOCSPResponse(time.Now().Add(-8 * 24 * time.Hour)) this.Require().NoError(err, "creating expired OCSP response") this.Assert().True(this.ocspServerCalled(func() { - _, _, err := this.handler.readOCSP() + _, _, err := this.handler.readOCSP(true) this.Require().NoError(err, "updating OCSP") })) // Verify that the invalid update doesn't squash the valid cache entry. - ocsp, _, err := this.handler.readOCSP() + ocsp, _, err := this.handler.readOCSP(true) this.Require().NoError(err, "reading OCSP") this.Assert().Equal(staleOCSP, ocsp) } +func (this *CertCacheSuite) TestPopulateCertCache() { + certCache, err := PopulateCertCache( + &util.Config{ + CertFile: "../../testdata/b3/fullchain.cert", + NewCertFile: "/tmp/newcert.cert", + KeyFile: "../../testdata/b3/server.privkey", + OCSPCache: "/tmp/ocsp", + URLSet: []util.URLSet{{ + Sign: &util.URLPattern{ + Domain: "amppackageexample.com", + PathRE: stringPtr(".*"), + QueryRE: stringPtr(""), + MaxLength: 2000, + }, + }}, + }, + pkgt.B3Key, + nil, + true, + false) + this.Require().NoError(err) + this.Assert().NotNil(certCache) + this.Assert().Equal(pkgt.B3Certs[0], certCache.GetLatestCert()) +} + func TestCertCacheSuite(t *testing.T) { suite.Run(t, new(CertCacheSuite)) } diff --git a/packager/certcache/storage.go b/packager/certcache/storage.go index beba184eb..7dbc381ea 100644 --- a/packager/certcache/storage.go +++ b/packager/certcache/storage.go @@ -8,8 +8,8 @@ import ( "runtime" "sync" - "github.com/pkg/errors" "github.com/gofrs/flock" + "github.com/pkg/errors" ) // This is an abstraction over a single file on a remote storage mechanism. It diff --git a/packager/certfetcher/certfetcher.go b/packager/certfetcher/certfetcher.go new file mode 100644 index 000000000..aa9447599 --- /dev/null +++ b/packager/certfetcher/certfetcher.go @@ -0,0 +1,170 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certfetcher + +import ( + "crypto" + "crypto/x509" + "strconv" + + "github.com/WICG/webpackage/go/signedexchange" + "github.com/go-acme/lego/v3/certcrypto" + "github.com/go-acme/lego/v3/challenge/http01" + "github.com/go-acme/lego/v3/challenge/tlsalpn01" + "github.com/go-acme/lego/v3/lego" + "github.com/go-acme/lego/v3/providers/dns" + "github.com/go-acme/lego/v3/providers/http/webroot" + "github.com/go-acme/lego/v3/registration" + "github.com/pkg/errors" +) + +type CertFetcher struct { + AcmeDiscoveryURL string + AcmeUser AcmeUser + legoClient *lego.Client + CertSignRequest *x509.CertificateRequest +} + +// Implements registration.User +type AcmeUser struct { + Email string + Registration *registration.Resource + key crypto.PrivateKey +} + +func (u *AcmeUser) GetEmail() string { + return u.Email +} +func (u AcmeUser) GetRegistration() *registration.Resource { + return u.Registration +} +func (u *AcmeUser) GetPrivateKey() crypto.PrivateKey { + return u.key +} + +// Initializes the cert fetcher with information it needs to fetch new certificates in the future. +// TODO(banaag): per gregable@ comments: +// Callsite could have some structure like: +// +// fetcher := CertFetcher() +// fetcher.setUser(email, privateKey) +// fetcher.bindToPort(port) +func New(email string, certSignRequest *x509.CertificateRequest, privateKey crypto.PrivateKey, + acmeDiscoURL string, httpChallengePort int, httpChallengeWebRoot string, + tlsChallengePort int, dnsProvider string, shouldRegister bool) (*CertFetcher, error) { + + acmeUser := AcmeUser{ + Email: email, + key: privateKey, + } + config := lego.NewConfig(&acmeUser) + + config.CADirURL = acmeDiscoURL + config.Certificate.KeyType = certcrypto.EC256 + + // A client facilitates communication with the CA server. + client, err := lego.NewClient(config) + if err != nil { + return nil, errors.Wrap(err, "Obtaining LEGO client.") + } + + // We specify an http port of `httpChallengePort` + // because we aren't running as root and can't bind a listener to port 80 and 443 + // (used later when we attempt to pass challenges). Keep in mind that you still + // need to proxy challenge traffic to port `acmeChallengePort`. + if httpChallengePort != 0 { + err := client.Challenge.SetHTTP01Provider( + http01.NewProviderServer("", strconv.Itoa(httpChallengePort))) + if err != nil { + return nil, errors.Wrap(err, "Setting up HTTP01 challenge provider.") + } + } + if httpChallengeWebRoot != "" { + httpProvider, err := webroot.NewHTTPProvider(httpChallengeWebRoot) + if err != nil { + return nil, errors.Wrap(err, "Getting HTTP01 challenge provider.") + } + err = client.Challenge.SetHTTP01Provider(httpProvider) + if err != nil { + return nil, errors.Wrap(err, "Setting up HTTP01 challenge provider.") + } + } + + if tlsChallengePort != 0 { + err := client.Challenge.SetTLSALPN01Provider(tlsalpn01.NewProviderServer("", strconv.Itoa(tlsChallengePort))) + if err != nil { + return nil, errors.Wrap(err, "Setting up TLSALPN01 challenge provider.") + } + } + + if dnsProvider != "" { + provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) + if err != nil { + return nil, errors.Wrap(err, "Getting DNS01 challenge provider.") + } + err = client.Challenge.SetDNS01Provider(provider) + if err != nil { + return nil, errors.Wrap(err, "Setting up DNS01 challenge provider.") + } + } + + // Theoretically, this should always be set to false as users should have pre-registered for access + // to the ACME CA and agreed to the TOS. + // TODO(banaag): revisit this when trying the class out with Digicert CA. + if !shouldRegister { + acmeUser.Registration = new(registration.Resource) + } else { + // TODO(banaag) make sure we present the TOS URL to the user and prompt for confirmation. + // The plan is to move this to some separate setup command outside the server which would be + // executed one time. Alternatively, we can have a field in the toml file that is documented + // to indicate agreement with TOS. + reg, err := client.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: true}) + if err != nil { + return nil, errors.Wrap(err, "ACME CA client registration") + } + acmeUser.Registration = reg + } + + return &CertFetcher{ + AcmeDiscoveryURL: acmeDiscoURL, + AcmeUser: acmeUser, + legoClient: client, + CertSignRequest: certSignRequest, + }, nil +} + +func (f *CertFetcher) FetchNewCert() ([]*x509.Certificate, error) { + // Each resource comes back with the cert bytes, the bytes of the client's + // private key, and a certificate URL. + resource, err := f.legoClient.Certificate.ObtainForCSR(*f.CertSignRequest, true) + if err != nil { + return nil, err + } + + if resource == nil { + return nil, errors.New("No resource returned.") + } + + if resource.Certificate == nil { + return nil, errors.New("No certificates were returned.") + } + + cert, err := signedexchange.ParseCertificates(resource.Certificate) + if err != nil { + return nil, err + } + + return cert, err +} diff --git a/packager/certfetcher/certfetcher_test.go b/packager/certfetcher/certfetcher_test.go new file mode 100644 index 000000000..e160c0a50 --- /dev/null +++ b/packager/certfetcher/certfetcher_test.go @@ -0,0 +1,246 @@ +package certfetcher + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "io/ioutil" + "net/http" + "testing" + + "github.com/go-acme/lego/v3/acme" + "github.com/go-acme/lego/v3/platform/tester" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + jose "gopkg.in/square/go-jose.v2" +) + +// CertResponseMock is just any valid SXG cert generated via: +// https://docs.digicert.com/manage-certificates/certificate-profile-options/get-your-signed-http-exchange-certificate/ +const CertResponseMock = `-----BEGIN CERTIFICATE----- +MIIFZDCCBOqgAwIBAgIQBxgJcqaHzEUEVOBlp+mFJTAKBggqhkjOPQQDAjBMMQsw +CQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSYwJAYDVQQDEx1EaWdp +Q2VydCBFQ0MgU2VjdXJlIFNlcnZlciBDQTAeFw0xODA4MzEwMDAwMDBaFw0yMDA5 +MDQxMjAwMDBaMHAxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMREw +DwYDVQQHEwhTYW4gSm9zZTEZMBcGA1UEChMQR3JlZ29yeSBHcm90aGF1czEeMBwG +A1UEAxMVYXplaS1wYWNrYWdlLXRlc3QuY29tMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAEc1QETMcI6mWbyAa4y026CLY/OnVGutWCrvTjO8WFZIZ16dxzO7UIsnPc +LdPVxnJQkY7uZnzfFYqLTBgHcgwE4KOCA4gwggOEMB8GA1UdIwQYMBaAFKOd5h/5 +2jlPwG7okcuVpdox4gqfMB0GA1UdDgQWBBQdgF7QKodsaqDTJ71z8z8hcXT0LTA7 +BgNVHREENDAyghVhemVpLXBhY2thZ2UtdGVzdC5jb22CGXd3dy5hemVpLXBhY2th +Z2UtdGVzdC5jb20wDgYDVR0PAQH/BAQDAgeAMB0GA1UdJQQWMBQGCCsGAQUFBwMB +BggrBgEFBQcDAjBpBgNVHR8EYjBgMC6gLKAqhihodHRwOi8vY3JsMy5kaWdpY2Vy +dC5jb20vc3NjYS1lY2MtZzEuY3JsMC6gLKAqhihodHRwOi8vY3JsNC5kaWdpY2Vy +dC5jb20vc3NjYS1lY2MtZzEuY3JsMEwGA1UdIARFMEMwNwYJYIZIAYb9bAEBMCow +KAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwCAYGZ4EM +AQIDMHsGCCsGAQUFBwEBBG8wbTAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZGln +aWNlcnQuY29tMEUGCCsGAQUFBzAChjlodHRwOi8vY2FjZXJ0cy5kaWdpY2VydC5j +b20vRGlnaUNlcnRFQ0NTZWN1cmVTZXJ2ZXJDQS5jcnQwDAYDVR0TAQH/BAIwADAQ +BgorBgEEAdZ5AgEWBAIFADCCAX4GCisGAQQB1nkCBAIEggFuBIIBagFoAHYApLkJ +kLQYWBSHuxOizGdwCjw1mAT5G9+443fNDsgN3BAAAAFlklqt1QAABAMARzBFAiAs +fo+czC5jBghS0acCZ8mLoMNFnnbvBnNCwXhIzohVQAIhALNlBFxwlJ7gift1XNtA +PLq3mI2vjtIpgtQ2azuVX/gBAHcAh3W/51l8+IxDmV+9827/Vo1HVjb/SrVgwbTq +/16ggw8AAAFlklquoQAABAMASDBGAiEAsG3VGlFzdghriY5qT3Mg3pEnLUHASVpu +bJXGfHBluUUCIQDIz36lErTWCOM5rJ7n5xWW15I1rumYCJzrUDZWjRShVgB1ALvZ +37wfinG1k5Qjl6qSe0c4V5UKq1LoGpCWZDaOHtGFAAABZZJardkAAAQDAEYwRAIg +S6oxgvn++wCfZ6wxt/lC2GoQX2LIJl5mrmHrMStgqxgCIF375hwD9aCMlv9SbfkL +GS2Mka/kMMtZrQVIQsyi3lhbMAoGCCqGSM49BAMCA2gAMGUCMBo9NIEu38bvGcKy +P9oN2ELBL3dgIXDq3oU85vX/8rEuwNLvsC4lMtk/QJap3dxuSAIxAMro/ZXw3lVP +YO0x/svBxXf6vDC01lO7LTJvjqA2Hfa/7GI5gbUr3sRTU09aO9ixOA== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDrDCCApSgAwIBAgIQCssoukZe5TkIdnRw883GEjANBgkqhkiG9w0BAQwFADBh +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD +QTAeFw0xMzAzMDgxMjAwMDBaFw0yMzAzMDgxMjAwMDBaMEwxCzAJBgNVBAYTAlVT +MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxJjAkBgNVBAMTHURpZ2lDZXJ0IEVDQyBT +ZWN1cmUgU2VydmVyIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4ghC6nfYJN6g +LGSkE85AnCNyqQIKDjc/ITa4jVMU9tWRlUvzlgKNcR7E2Munn17voOZ/WpIRllNv +68DLP679Wz9HJOeaBy6Wvqgvu1cYr3GkvXg6HuhbPGtkESvMNCuMo4IBITCCAR0w +EgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwNAYIKwYBBQUHAQEE +KDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wQgYDVR0f +BDswOTA3oDWgM4YxaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0R2xv +YmFsUm9vdENBLmNybDA9BgNVHSAENjA0MDIGBFUdIAAwKjAoBggrBgEFBQcCARYc +aHR0cHM6Ly93d3cuZGlnaWNlcnQuY29tL0NQUzAdBgNVHQ4EFgQUo53mH/naOU/A +buiRy5Wl2jHiCp8wHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDQYJ +KoZIhvcNAQEMBQADggEBAMeKoENL7HTJxavVHzA1Nm6YVntIrAVjrnuaVyRXzG/6 +3qttnMe2uuzO58pzZNvfBDcKAEmzP58mrZGMIOgfiA4q+2Y3yDDo0sIkp0VILeoB +UEoxlBPfjV/aKrtJPGHzecicZpIalir0ezZYoyxBEHQa0+1IttK7igZFcTMQMHp6 +mCHdJLnsnLWSB62DxsRq+HfmNb4TDydkskO/g+l3VtsIh5RHFPVfKK+jaEyDj2D3 +loB5hWp2Jp2VDCADjT7ueihlZGak2YPqmXTNbk19HOuNssWvFhtOyPNV6og4ETQd +Ea8/B6hPatJ0ES8q/HO3X8IVQwVs1n3aAr0im0/T+Xc= +-----END CERTIFICATE----- +` + +func TestNewFetcher(t *testing.T) { + _, apiURL, tearDown := tester.SetupFakeAPI() + defer tearDown() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "Could not generate test key") + + csr := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + DNSNames: []string{"test.example.com"}, + } + + fetcher, err := New("test@test.com", &csr, privateKey, apiURL+"/dir", + 5002, "", 0, "", false) + assert.Nil(t, err) + assert.NotNil(t, fetcher.legoClient) + assert.Equal(t, "test@test.com", fetcher.AcmeUser.Email) + assert.Equal(t, privateKey, fetcher.AcmeUser.key) +} + +func TestFetchCertSuccess(t *testing.T) { + mux, apiURL, tearDown := tester.SetupFakeAPI() + defer tearDown() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "Could not generate test key") + + setupMux(mux, apiURL, privateKey) + mux.HandleFunc("/certificate", func(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte(CertResponseMock)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + }) + + csr := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + DNSNames: []string{"test.example.com"}, + } + + fetcher, err := New("test@test.com", &csr, privateKey, apiURL+"/dir", + 5002, "", 0, "", false) + assert.Nil(t, err) + assert.NotNil(t, fetcher) + + cert, err := fetcher.FetchNewCert() + assert.Nil(t, err) + assert.NotNil(t, cert) +} + +func TestFetchCertFail(t *testing.T) { + mux, apiURL, tearDown := tester.SetupFakeAPI() + defer tearDown() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err, "Could not generate test key") + + setupMux(mux, apiURL, privateKey) + mux.HandleFunc("/certificate", func(w http.ResponseWriter, _ *http.Request) { + // Intentionally return an error. + http.Error(w, "", http.StatusInternalServerError) + }) + + csr := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "test.example.com", + Organization: []string{"Acme Co"}, + }, + DNSNames: []string{"test.example.com"}, + } + + fetcher, err := New("test@test.com", &csr, privateKey, apiURL+"/dir", + 5002, "", 0, "", false) + assert.Nil(t, err) + assert.NotNil(t, fetcher) + + cert, err := fetcher.FetchNewCert() + assert.NotNil(t, err) + assert.Nil(t, cert) +} + +func setupMux(mux *http.ServeMux, apiURL string, privateKey *rsa.PrivateKey) { + mux.HandleFunc("/newOrder", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + return + } + + body, err := readSignedBody(r, privateKey) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + order := acme.Order{} + err = json.Unmarshal(body, &order) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err = tester.WriteJSONResponse(w, acme.Order{ + Status: acme.StatusValid, + Finalize: apiURL + "/finalize", + Identifiers: order.Identifiers, + }) + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + mux.HandleFunc("/finalize", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) + return + } + + body, err := readSignedBody(r, privateKey) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + order := acme.Order{} + err = json.Unmarshal(body, &order) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + err = tester.WriteJSONResponse(w, acme.Order{ + Status: acme.StatusValid, + Identifiers: order.Identifiers, + Certificate: apiURL + "/certificate", + }) + + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) +} + +func readSignedBody(r *http.Request, privateKey *rsa.PrivateKey) ([]byte, error) { + reqBody, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + + jws, err := jose.ParseSigned(string(reqBody)) + if err != nil { + return nil, err + } + + body, err := jws.Verify(&jose.JSONWebKey{ + Key: privateKey.Public(), + Algorithm: "RSA", + }) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/packager/certloader/certloader.go b/packager/certloader/certloader.go new file mode 100644 index 000000000..79dfae904 --- /dev/null +++ b/packager/certloader/certloader.go @@ -0,0 +1,256 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certloader + +import ( + "crypto" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "log" + "os" + + "github.com/WICG/webpackage/go/signedexchange" + "github.com/gofrs/flock" + "github.com/pkg/errors" + + "github.com/ampproject/amppackager/packager/certfetcher" + "github.com/ampproject/amppackager/packager/util" +) + +func CreateCertFetcher(config *util.Config, key crypto.PrivateKey, domain string, + developmentMode bool, autoRenewCert bool) (*certfetcher.CertFetcher, error) { + if !autoRenewCert { + // Certfetcher can be nil, if auto renew is off. + return nil, nil + } + + if config.ACMEConfig == nil { + return nil, errors.New("missing ACMEConfig") + } + + var acmeConfig *util.ACMEServerConfig + if developmentMode { + acmeConfig = config.ACMEConfig.Development + } else { + acmeConfig = config.ACMEConfig.Production + } + + if acmeConfig == nil { + if developmentMode { + return nil, errors.New("missing ACMEConfig.Development") + } else { + return nil, errors.New("missing ACMEConfig.Production") + } + } + + if acmeConfig.EmailAddress == "" { + return nil, errors.New("missing email address") + } + emailAddress := acmeConfig.EmailAddress + if acmeConfig.DiscoURL == "" { + return nil, errors.New("missing acme disco url") + } + acmeDiscoveryURL := acmeConfig.DiscoURL + if acmeConfig.HttpChallengePort == 0 && + acmeConfig.HttpWebRootDir == "" && + acmeConfig.TlsChallengePort == 0 && + acmeConfig.DnsProvider == "" { + return nil, errors.New("One of HttpChallengePort, HttpWebRootDir, TlsChallengePort and DnsProvider must be present.") + } + httpChallengePort := acmeConfig.HttpChallengePort + httpWebRootDir := acmeConfig.HttpWebRootDir + tlsChallengePort := acmeConfig.TlsChallengePort + dnsProvider := acmeConfig.DnsProvider + + // TODO(banaag): Rather than making publishers create a CSR, generate one using the given KeyFile/CertFile and + // https://golang.org/pkg/crypto/x509/#CreateCertificateRequest. + csr, err := LoadCSRFromFile(config) + if err != nil { + return nil, errors.Wrap(err, "missing CSR") + } + + // Create the cert fetcher that will auto-renew the cert. + certFetcher, err := certfetcher.New(emailAddress, csr, key, acmeDiscoveryURL, + httpChallengePort, httpWebRootDir, tlsChallengePort, dnsProvider, true) + if err != nil { + return nil, errors.Wrap(err, "creating certfetcher") + } + log.Println("Certfetcher created successfully.") + return certFetcher, nil +} + +// Loads X509 certificates from disk. +// Returns appropriate errors if: +// The file can't be read. +// The certificate can't be parsed. +// No certificates found in the file. +// Certificates cannot be used to sign HTTP exchanges. +// (if developmentMode, print a warning that certs can't +// be used to sign HTTP exchanges). +// If there are no errors, the array of certificates is returned. +func LoadCertsFromFile(config *util.Config, developmentMode bool) ([]*x509.Certificate, error) { + return LoadAndValidateCertsFromFile(config.CertFile, !developmentMode) +} + +func LoadAndValidateCertsFromFile(certPath string, requireSign bool) ([]*x509.Certificate, error) { + // Use independent .lock file; necessary on Windows to avoid "The process cannot + // access the file because another process has locked a portion of the file." + lockPath := certPath + ".lock" + lock := flock.New(lockPath) + locked, err := lock.TryRLock() + if err != nil { + return nil, errors.Wrapf(err, "obtaining exclusive lock for %s", lockPath) + } + if !locked { + return nil, errors.Errorf("unable to obtain exclusive lock for %s", lockPath) + } + defer func() { + if err = lock.Unlock(); err != nil { + log.Printf("Error unlocking %s; %+v", lockPath, err) + } + if err := os.Remove(lockPath); err != nil { + log.Printf("Error removing %s; %+v", lockPath, err) + } + }() + + // TODO(twifkak): Document what cert/key storage formats this accepts. + certPem, err := ioutil.ReadFile(certPath) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", certPath) + } + certs, err := signedexchange.ParseCertificates(certPem) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s", certPath) + } + if certs == nil || len(certs) == 0 { + return nil, errors.Errorf("no cert found in %s", certPath) + } + if err := util.CanSignHttpExchanges(certs[0]); err != nil { + if !requireSign { + log.Println("WARNING:", err) + } else { + return nil, err + } + } + + return certs, nil +} + +func WriteCertsToFile(certs []*x509.Certificate, filepath string) error { + if len(certs) < 2 { + return errors.New("Missing issuer in bundle") + } + + // Use independent .lock file; necessary on Windows to avoid "The process cannot + // access the file because another process has locked a portion of the file." + lockPath := filepath + ".lock" + lock := flock.New(lockPath) + locked, err := lock.TryLock() + if err != nil { + return errors.Wrapf(err, "obtaining exclusive lock for %s", lockPath) + } + if !locked { + return errors.Errorf("unable to obtain exclusive lock for %s", lockPath) + } + defer func() { + if err = lock.Unlock(); err != nil { + log.Printf("Error unlocking %s; %+v", lockPath, err) + } + if err := os.Remove(lockPath); err != nil { + log.Printf("Error removing %s; %+v", lockPath, err) + } + }() + + bundled := []byte{} + for _, cert := range certs { + pem := certToPEM(cert) + bundled = append(bundled, pem...) + } + if err := ioutil.WriteFile(filepath, bundled, 0600); err != nil { + return errors.Wrapf(err, "writing %s", filepath) + } + + return nil +} + +func RemoveFile(filepath string) error { + // Use independent .lock file; necessary on Windows to avoid "The process cannot + // access the file because another process has locked a portion of the file." + lockPath := filepath + ".lock" + lock := flock.New(lockPath) + locked, err := lock.TryLock() + if err != nil { + return errors.Wrapf(err, "obtaining exclusive lock for %s", lockPath) + } + if !locked { + return errors.Errorf("unable to obtain exclusive lock for %s", lockPath) + } + defer func() { + if err = lock.Unlock(); err != nil { + log.Printf("Error unlocking %s; %+v", lockPath, err) + } + if err := os.Remove(lockPath); err != nil { + log.Printf("Error removing %s; %+v", lockPath, err) + } + }() + + if err := os.Remove(filepath); err != nil { + return errors.Wrapf(err, "removing %s", filepath) + } + + return nil +} + +func certToPEM(cert *x509.Certificate) []byte { + pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + + return pemCert +} + +func LoadCSRFromFile(config *util.Config) (*x509.CertificateRequest, error) { + data, err := ioutil.ReadFile(config.CSRFile) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", config.CSRFile) + } + block, _ := pem.Decode(data) + if block == nil { + return nil, errors.Errorf("pem decode: no key found in %s", config.CSRFile) + } + csr, err := x509.ParseCertificateRequest(block.Bytes) + if err != nil { + return nil, errors.Wrapf(err, "parsing CSR %s", config.CSRFile) + } + return csr, nil +} + +// Loads private key from file. +// Returns appropriate errors if: +// The file can't be read. +// The key can't be parsed. +// If there are no errors, the key is returned. +func LoadKeyFromFile(config *util.Config) (crypto.PrivateKey, error) { + keyPem, err := ioutil.ReadFile(config.KeyFile) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", config.KeyFile) + } + + key, err := util.ParsePrivateKey(keyPem) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s", config.KeyFile) + } + + return key, nil +} diff --git a/packager/certloader/certloader_test.go b/packager/certloader/certloader_test.go new file mode 100644 index 000000000..457ac470c --- /dev/null +++ b/packager/certloader/certloader_test.go @@ -0,0 +1,88 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certloader + +import ( + "crypto/rsa" + "crypto/x509" + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/WICG/webpackage/go/signedexchange" + "github.com/ampproject/amppackager/packager/util" +) + +func stringPtr(s string) *string { + return &s +} + +var caCert = func() *x509.Certificate { + certPem, _ := ioutil.ReadFile("../../testdata/b3/ca.cert") + certs, _ := signedexchange.ParseCertificates(certPem) + return certs[0] +}() + +var caKey = func() *rsa.PrivateKey { + keyPem, _ := ioutil.ReadFile("../../testdata/b3/ca.privkey") + key, _ := util.ParsePrivateKey(keyPem) + return key.(*rsa.PrivateKey) +}() + +func TestLoadCertsFromFile(t *testing.T) { + // Cert file does not exist. + certs, err := LoadCertsFromFile( + &util.Config{ + CertFile: "file_does_not_exist", + }, + true) + assert.Contains(t, err.Error(), "no such file or directory") + + // Cert file is ok for dev mode. + certs, err = LoadCertsFromFile( + &util.Config{ + CertFile: "../../testdata/b3/ca.cert", + }, + true) + assert.Equal(t, caCert, certs[0]) + assert.Nil(t, err) + + // Cert file is not ok for prod mode. + certs, err = LoadCertsFromFile( + &util.Config{ + CertFile: "../../testdata/b3/ca.cert", + }, + false) + assert.Equal(t, certs, ([]*x509.Certificate)(nil)) + assert.Equal(t, err.Error(), "Certificate is missing CanSignHttpExchanges extension") +} + +func TestLoadKeyFromFile(t *testing.T) { + // Key does not exist. + key, err := LoadKeyFromFile( + &util.Config{ + KeyFile: "file_does_not_exist", + }) + assert.Contains(t, err.Error(), "no such file or directory") + + // Key is valid. + key, err = LoadKeyFromFile( + &util.Config{ + KeyFile: "../../testdata/b3/ca.privkey", + }) + assert.Equal(t, caKey, key) + assert.Nil(t, err) +} diff --git a/packager/healthz/healthz.go b/packager/healthz/healthz.go new file mode 100644 index 000000000..911711af3 --- /dev/null +++ b/packager/healthz/healthz.go @@ -0,0 +1,41 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthz + +import ( + "fmt" + "github.com/ampproject/amppackager/packager/certcache" + "net/http" +) + +type Healthz struct { + certHandler certcache.CertHandler +} + +func New(certHandler certcache.CertHandler) (*Healthz, error) { + return &Healthz{certHandler}, nil +} + +func (this *Healthz) ServeHTTP(resp http.ResponseWriter, req *http.Request) { + // Follow https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/ + err := this.certHandler.IsHealthy() + if err != nil { + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf("not healthy: %v", err))) + } else { + resp.WriteHeader(200) + resp.Write([]byte("ok")) + } +} diff --git a/packager/healthz/healthz_test.go b/packager/healthz/healthz_test.go new file mode 100644 index 000000000..3d25cb143 --- /dev/null +++ b/packager/healthz/healthz_test.go @@ -0,0 +1,64 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package healthz + +import ( + "crypto/x509" + "net/http" + "testing" + + "github.com/ampproject/amppackager/packager/mux" + "github.com/pkg/errors" + + pkgt "github.com/ampproject/amppackager/packager/testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeHealthyCertHandler struct { +} + +func (this fakeHealthyCertHandler) GetLatestCert() *x509.Certificate { + return pkgt.Certs[0] +} + +func (this fakeHealthyCertHandler) IsHealthy() error { + return nil +} + +type fakeNotHealthyCertHandler struct { +} + +func (this fakeNotHealthyCertHandler) GetLatestCert() *x509.Certificate { + return pkgt.Certs[0] +} + +func (this fakeNotHealthyCertHandler) IsHealthy() error { + return errors.New("random error") +} + +func TestHealthzOk(t *testing.T) { + handler, err := New(fakeHealthyCertHandler{}) + require.NoError(t, err) + resp := pkgt.Get(t, mux.New(nil, nil, nil, handler), "/healthz") + assert.Equal(t, http.StatusOK, resp.StatusCode, "ok", resp) +} + +func TestHealthzFail(t *testing.T) { + handler, err := New(fakeNotHealthyCertHandler{}) + require.NoError(t, err) + resp := pkgt.Get(t, mux.New(nil, nil, nil, handler), "/healthz") + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode, "error", resp) +} diff --git a/packager/mux/mux.go b/packager/mux/mux.go index b7d8e8ddd..e5696c272 100644 --- a/packager/mux/mux.go +++ b/packager/mux/mux.go @@ -45,11 +45,12 @@ type mux struct { certCache http.Handler signer http.Handler validityMap http.Handler + healthz http.Handler } // The main entry point. Use the return value for http.Server.Handler. -func New(certCache http.Handler, signer http.Handler, validityMap http.Handler) http.Handler { - return &mux{certCache, signer, validityMap} +func New(certCache http.Handler, signer http.Handler, validityMap http.Handler, healthz http.Handler) http.Handler { + return &mux{certCache, signer, validityMap, healthz} } func tryTrimPrefix(s, prefix string) (string, bool) { @@ -97,6 +98,8 @@ func (this *mux) ServeHTTP(resp http.ResponseWriter, req *http.Request) { params["certName"] = unescaped this.certCache.ServeHTTP(resp, req) } + } else if path == util.HealthzPath { + this.healthz.ServeHTTP(resp, req) } else if path == util.ValidityMapPath { this.validityMap.ServeHTTP(resp, req) } else { diff --git a/packager/signer/signer.go b/packager/signer/signer.go index 8a167ef10..5b3c54c37 100644 --- a/packager/signer/signer.go +++ b/packager/signer/signer.go @@ -32,6 +32,7 @@ import ( "github.com/WICG/webpackage/go/signedexchange" "github.com/ampproject/amppackager/packager/accept" "github.com/ampproject/amppackager/packager/amp_cache_transform" + "github.com/ampproject/amppackager/packager/certcache" "github.com/ampproject/amppackager/packager/mux" "github.com/ampproject/amppackager/packager/rtv" "github.com/ampproject/amppackager/packager/util" @@ -121,15 +122,15 @@ type Signer struct { // TODO(twifkak): Support multiple certs. This will require generating // a signature for each one. Note that Chrome only supports 1 signature // at the moment. - cert *x509.Certificate + certHandler certcache.CertHandler // TODO(twifkak): Do we want to allow multiple keys? - key crypto.PrivateKey - client *http.Client - urlSets []util.URLSet - rtvCache *rtv.RTVCache - shouldPackage func() bool - overrideBaseURL *url.URL - requireHeaders bool + key crypto.PrivateKey + client *http.Client + urlSets []util.URLSet + rtvCache *rtv.RTVCache + shouldPackage func() error + overrideBaseURL *url.URL + requireHeaders bool forwardedRequestHeaders []string } @@ -137,8 +138,8 @@ func noRedirects(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } -func New(cert *x509.Certificate, key crypto.PrivateKey, urlSets []util.URLSet, - rtvCache *rtv.RTVCache, shouldPackage func() bool, overrideBaseURL *url.URL, +func New(certHandler certcache.CertHandler, key crypto.PrivateKey, urlSets []util.URLSet, + rtvCache *rtv.RTVCache, shouldPackage func() error, overrideBaseURL *url.URL, requireHeaders bool, forwardedRequestHeaders []string) (*Signer, error) { client := http.Client{ CheckRedirect: noRedirects, @@ -146,7 +147,7 @@ func New(cert *x509.Certificate, key crypto.PrivateKey, urlSets []util.URLSet, Timeout: 60 * time.Second, } - return &Signer{cert, key, &client, urlSets, rtvCache, shouldPackage, overrideBaseURL, requireHeaders, forwardedRequestHeaders}, nil + return &Signer{certHandler, key, &client, urlSets, rtvCache, shouldPackage, overrideBaseURL, requireHeaders, forwardedRequestHeaders}, nil } func (this *Signer) fetchURL(fetch *url.URL, serveHTTPReq *http.Request) (*http.Request, *http.Response, *util.HTTPError) { @@ -176,6 +177,17 @@ func (this *Signer) fetchURL(fetch *url.URL, serveHTTPReq *http.Request) (*http. } req.Header.Set("Via", via) } + if quotedHost, err := util.QuotedString(serveHTTPReq.Host); err == nil { + // TODO(twifkak): Extract host from upstream Forwarded header + // and concatenate. (Do not include any other parameters, as + // they may lead to over-signing.) + req.Header.Set("Forwarded", `host=` + quotedHost) + xfh := serveHTTPReq.Host + if oldXFH := serveHTTPReq.Header.Get("X-Forwarded-Host"); oldXFH != "" { + xfh = oldXFH + "," + xfh + } + req.Header.Set("X-Forwarded-Host", xfh) + } // Set conditional headers that were included in ServeHTTP's Request. for header := range util.ConditionalRequestHeaders { if value := GetJoined(serveHTTPReq.Header, header); value != "" { @@ -306,8 +318,8 @@ func (this *Signer) ServeHTTP(resp http.ResponseWriter, req *http.Request) { } }() - if !this.shouldPackage() { - log.Println("Not packaging because server is unhealthy; see above log statements.") + if err := this.shouldPackage(); err != nil { + log.Println("Not packaging because server is unhealthy; see above log statements.", err) proxy(resp, fetchResp, nil) return } @@ -459,13 +471,14 @@ func (this *Signer) serveSignedExchange(resp http.ResponseWriter, fetchResp *htt fetchResp.Header.Get("Content-Security-Policy"))) exchange := signedexchange.NewExchange( - accept.SxgVersion, /*uri=*/signURL.String(), /*method=*/"GET", + accept.SxgVersion /*uri=*/, signURL.String() /*method=*/, "GET", http.Header{}, fetchResp.StatusCode, fetchResp.Header, []byte(transformed)) if err := exchange.MiEncodePayload(miRecordSize); err != nil { util.NewHTTPError(http.StatusInternalServerError, "Error MI-encoding: ", err).LogAndRespond(resp) return } - certURL, err := this.genCertURL(this.cert, signURL) + cert := this.certHandler.GetLatestCert() + certURL, err := this.genCertURL(cert, signURL) if err != nil { util.NewHTTPError(http.StatusInternalServerError, "Error building cert URL: ", err).LogAndRespond(resp) return @@ -475,12 +488,17 @@ func (this *Signer) serveSignedExchange(resp http.ResponseWriter, fetchResp *htt if err != nil { util.NewHTTPError(http.StatusInternalServerError, "Error building validity href: ", err).LogAndRespond(resp) } + // Expires - Date must be <= 604800 seconds, per + // https://tools.ietf.org/html/draft-yasskin-httpbis-origin-signed-exchanges-impl-00#section-3.5. + duration := 7 * 24 * time.Hour + if maxAge := time.Duration(metadata.MaxAgeSecs) * time.Second; maxAge < duration { + duration = maxAge + } + date := now.Add(-24 * time.Hour) signer := signedexchange.Signer{ - // Expires - Date must be <= 604800 seconds, per - // https://tools.ietf.org/html/draft-yasskin-httpbis-origin-signed-exchanges-impl-00#section-3.5. - Date: now.Add(-24 * time.Hour), - Expires: now.Add(6 * 24 * time.Hour), - Certs: []*x509.Certificate{this.cert}, + Date: date, + Expires: date.Add(duration), + Certs: []*x509.Certificate{cert}, CertUrl: certURL, ValidityUrl: signURL.ResolveReference(validityHRef), PrivKey: this.key, @@ -500,7 +518,7 @@ func (this *Signer) serveSignedExchange(resp http.ResponseWriter, fetchResp *htt // If requireHeaders was true when constructing signer, the // AMP-Cache-Transform outer response header is required (and has already // been validated) - if (act != "") { + if act != "" { resp.Header().Set("AMP-Cache-Transform", act) } diff --git a/packager/signer/signer_test.go b/packager/signer/signer_test.go index a00d15f97..67add030b 100644 --- a/packager/signer/signer_test.go +++ b/packager/signer/signer_test.go @@ -16,6 +16,8 @@ package signer import ( "bytes" + "crypto/x509" + "encoding/base64" "encoding/binary" "fmt" "io/ioutil" @@ -27,6 +29,7 @@ import ( "testing" "github.com/WICG/webpackage/go/signedexchange" + "github.com/WICG/webpackage/go/signedexchange/structuredheader" "github.com/ampproject/amppackager/packager/accept" "github.com/ampproject/amppackager/packager/mux" "github.com/ampproject/amppackager/packager/rtv" @@ -34,6 +37,7 @@ import ( "github.com/ampproject/amppackager/packager/util" "github.com/ampproject/amppackager/transformer" rpb "github.com/ampproject/amppackager/transformer/request" + "github.com/pkg/errors" "github.com/stretchr/testify/suite" ) @@ -52,22 +56,33 @@ func headerNames(headers http.Header) []string { return names } +type fakeCertHandler struct { +} + +func (this fakeCertHandler) GetLatestCert() *x509.Certificate { + return pkgt.Certs[0] +} + +func (this fakeCertHandler) IsHealthy() error { + return nil +} + type SignerSuite struct { suite.Suite httpServer, tlsServer *httptest.Server httpsClient *http.Client - shouldPackage bool + shouldPackage error fakeHandler func(resp http.ResponseWriter, req *http.Request) lastRequest *http.Request } func (this *SignerSuite) new(urlSets []util.URLSet) http.Handler { forwardedRequestHeaders := []string{"Host", "X-Foo"} - handler, err := New(pkgt.Certs[0], pkgt.Key, urlSets, &rtv.RTVCache{}, func() bool { return this.shouldPackage }, nil, true, forwardedRequestHeaders) + handler, err := New(fakeCertHandler{}, pkgt.Key, urlSets, &rtv.RTVCache{}, func() error { return this.shouldPackage }, nil, true, forwardedRequestHeaders) this.Require().NoError(err) // Accept the self-signed certificate generated by the test server. handler.client = this.httpsClient - return mux.New(nil, handler, nil) + return mux.New(nil, handler, nil, nil) } func (this *SignerSuite) get(t *testing.T, handler http.Handler, target string) *http.Response { @@ -143,7 +158,7 @@ func (this *SignerSuite) TearDownSuite() { } func (this *SignerSuite) SetupTest() { - this.shouldPackage = true + this.shouldPackage = nil this.fakeHandler = func(resp http.ResponseWriter, req *http.Request) { this.lastRequest = req resp.Header().Set("Content-Type", "text/html") @@ -168,6 +183,8 @@ func (this *SignerSuite) TestSimple() { this.Assert().Equal(fakePath, this.lastRequest.URL.String()) this.Assert().Equal(userAgent, this.lastRequest.Header.Get("User-Agent")) this.Assert().Equal("1.1 amppkg", this.lastRequest.Header.Get("Via")) + this.Assert().Equal(`host="example.com"`, this.lastRequest.Header.Get("Forwarded")) + this.Assert().Equal("example.com", this.lastRequest.Header.Get("X-Forwarded-Host")) this.Assert().Equal(http.StatusOK, resp.StatusCode, "incorrect status: %#v", resp) this.Assert().Equal(fmt.Sprintf(`google;v="%d"`, transformer.SupportedVersions[0].Max), resp.Header.Get("AMP-Cache-Transform")) this.Assert().Equal("nosniff", resp.Header.Get("X-Content-Type-Options")) @@ -187,8 +204,19 @@ func (this *SignerSuite) TestSimple() { this.Assert().Contains(exchange.SignatureHeaderValue, "validity-url=\""+this.httpSignURL()+"/amppkg/validity\"") this.Assert().Contains(exchange.SignatureHeaderValue, "integrity=\"digest/mi-sha256-03\"") this.Assert().Contains(exchange.SignatureHeaderValue, "cert-url=\""+this.httpSignURL()+"/amppkg/cert/"+pkgt.CertName+"\"") - this.Assert().Contains(exchange.SignatureHeaderValue, "cert-sha256=*"+pkgt.CertName+"=*") + certHash, _ := base64.RawURLEncoding.DecodeString(pkgt.CertName) + this.Assert().Contains(exchange.SignatureHeaderValue, "cert-sha256=*"+base64.StdEncoding.EncodeToString(certHash[:])+"*") // TODO(twifkak): Control date, and test for expires and sig. + + signatures, err := structuredheader.ParseParameterisedList(exchange.SignatureHeaderValue) + this.Require().NoError(err) + this.Require().NotEmpty(signatures) + date, ok := signatures[0].Params["date"].(int64) + this.Require().True(ok) + expires, ok := signatures[0].Params["expires"].(int64) + this.Require().True(ok) + this.Assert().Equal(int64(604800), expires-date) + // The response header values are untested here, as that is covered by signedexchange tests. // For small enough bodies, the only thing that MICE does is add a record size prefix. @@ -239,12 +267,32 @@ func (this *SignerSuite) TestFetchSignWithForwardedRequestHeaders() { this.Assert().Contains(exchange.SignatureHeaderValue, "validity-url=\""+this.httpSignURL_CertSubjectCN()+"/amppkg/validity\"") this.Assert().Contains(exchange.SignatureHeaderValue, "integrity=\"digest/mi-sha256-03\"") this.Assert().Contains(exchange.SignatureHeaderValue, "cert-url=\""+this.httpSignURL_CertSubjectCN()+"/amppkg/cert/"+pkgt.CertName+"\"") - this.Assert().Contains(exchange.SignatureHeaderValue, "cert-sha256=*"+pkgt.CertName+"=*") + certHash, _ := base64.RawURLEncoding.DecodeString(pkgt.CertName) + this.Assert().Contains(exchange.SignatureHeaderValue, "cert-sha256=*"+base64.StdEncoding.EncodeToString(certHash[:])+"*") var payloadPrefix bytes.Buffer binary.Write(&payloadPrefix, binary.BigEndian, uint64(miRecordSize)) this.Assert().Equal(append(payloadPrefix.Bytes(), transformedBody...), exchange.Payload) } +func (this *SignerSuite) TestForwardedHost() { + urlSets := []util.URLSet{{ + Sign: &util.URLPattern{[]string{"https"}, "", this.httpHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, nil}, + Fetch: &util.URLPattern{[]string{"http"}, "", this.httpHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, boolPtr(true)}, + }} + header := http.Header{ + "AMP-Cache-Transform": {"google"}, "Accept": {"application/signed-exchange;v=" + accept.AcceptedSxgVersion}, + "Forwarded": {`host="www.example.com";for=192.0.0.1`}, + "X-Forwarded-For": {"192.0.0.1"}, + "X-Forwarded-Host": {"www.example.com"}} + this.getFRH(this.T(), this.new(urlSets), + "/priv/doc?fetch="+url.QueryEscape(this.httpURL()+fakePath)+ + "&sign="+url.QueryEscape(this.httpSignURL()+fakePath), + "example.com", header) + + this.Assert().Equal(`host="example.com"`, this.lastRequest.Header.Get("Forwarded")) + this.Assert().Equal("www.example.com,example.com", this.lastRequest.Header.Get("X-Forwarded-Host")) +} + func (this *SignerSuite) TestEscapeQueryParamsInFetchAndSign() { urlSets := []util.URLSet{{ Sign: &util.URLPattern{[]string{"https"}, "", this.httpHost(), stringPtr("/amp/.*"), []string{}, stringPtr(".*"), false, 2000, nil}, @@ -394,6 +442,7 @@ func (this *SignerSuite) TestMutatesCspHeaders() { "script-src https://notallowed.org/") resp.Write(fakeBody) } + resp := this.get( this.T(), this.new(urlSets), @@ -471,6 +520,50 @@ func (this *SignerSuite) TestRemovesHopByHopHeaders() { this.Assert().NotContains(exchange.ResponseHeaders, http.CanonicalHeaderKey("Transfer-Encoding")) } +func (this *SignerSuite) TestLimitsDuration() { + urlSets := []util.URLSet{{ + Sign: &util.URLPattern{[]string{"https"}, "", this.httpsHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, nil}}} + this.fakeHandler = func(resp http.ResponseWriter, req *http.Request) { + resp.Header().Set("Content-Type", "text/html; charset=utf-8") + resp.Write([]byte("")) + } + resp := this.get(this.T(), this.new(urlSets), "/priv/doc?sign="+url.QueryEscape(this.httpsURL()+fakePath)) + this.Assert().Equal(http.StatusOK, resp.StatusCode, "incorrect status: %#v", resp) + + exchange, err := signedexchange.ReadExchange(resp.Body) + this.Require().NoError(err) + signatures, err := structuredheader.ParseParameterisedList(exchange.SignatureHeaderValue) + this.Require().NoError(err) + this.Require().NotEmpty(signatures) + date, ok := signatures[0].Params["date"].(int64) + this.Require().True(ok) + expires, ok := signatures[0].Params["expires"].(int64) + this.Require().True(ok) + this.Assert().Equal(int64(4000), expires-date) +} + +func (this *SignerSuite) TestDoesNotExtendDuration() { + urlSets := []util.URLSet{{ + Sign: &util.URLPattern{[]string{"https"}, "", this.httpsHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, nil}}} + this.fakeHandler = func(resp http.ResponseWriter, req *http.Request) { + resp.Header().Set("Content-Type", "text/html; charset=utf-8") + resp.Write([]byte("")) + } + resp := this.get(this.T(), this.new(urlSets), "/priv/doc?sign="+url.QueryEscape(this.httpsURL()+fakePath)) + this.Assert().Equal(http.StatusOK, resp.StatusCode, "incorrect status: %#v", resp) + + exchange, err := signedexchange.ReadExchange(resp.Body) + this.Require().NoError(err) + signatures, err := structuredheader.ParseParameterisedList(exchange.SignatureHeaderValue) + this.Require().NoError(err) + this.Require().NotEmpty(signatures) + date, ok := signatures[0].Params["date"].(int64) + this.Require().True(ok) + expires, ok := signatures[0].Params["expires"].(int64) + this.Require().True(ok) + this.Assert().Equal(int64(604800), expires-date) +} + func (this *SignerSuite) TestErrorNoCache() { urlSets := []util.URLSet{{ Fetch: &util.URLPattern{[]string{"http"}, "", this.httpHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, boolPtr(true)}, @@ -521,7 +614,7 @@ func (this *SignerSuite) TestProxyUnsignedIfShouldntPackage() { urlSets := []util.URLSet{{ Sign: &util.URLPattern{[]string{"https"}, "", this.httpsHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, nil}, }} - this.shouldPackage = false + this.shouldPackage = errors.New("random error") resp := this.get(this.T(), this.new(urlSets), "/priv/doc?sign="+url.QueryEscape(this.httpsURL()+fakePath)) this.Assert().Equal(http.StatusOK, resp.StatusCode, "incorrect status: %#v", resp) body, err := ioutil.ReadAll(resp.Body) @@ -546,7 +639,7 @@ func (this *SignerSuite) TestProxyUnsignedIfInvalidAMPCacheTransformHeader() { Sign: &util.URLPattern{[]string{"https"}, "", this.httpsHost(), stringPtr("/amp/.*"), []string{}, stringPtr(""), false, 2000, nil}, }} resp := pkgt.GetH(this.T(), this.new(urlSets), "/priv/doc?sign="+url.QueryEscape(this.httpsURL()+fakePath), http.Header{ - "Accept": {"application/signed-exchange;v=" + accept.AcceptedSxgVersion}, + "Accept": {"application/signed-exchange;v=" + accept.AcceptedSxgVersion}, "AMP-Cache-Transform": {"donotmatch"}, }) this.Assert().Equal(http.StatusOK, resp.StatusCode, "incorrect status: %#v", resp) @@ -716,10 +809,10 @@ func (this *SignerSuite) TestProxyHeadersUnaltered() { Transformers: []string{"bogus"}} } - originalHeaders := map[string]string { - "Content-Type": "text/html", - "Set-Cookie": "chocolate chip", - "Cache-Control": "max-age=31536000", + originalHeaders := map[string]string{ + "Content-Type": "text/html", + "Set-Cookie": "chocolate chip", + "Cache-Control": "max-age=31536000", "Content-Length": fmt.Sprintf("%d", len(fakeBody)), } diff --git a/packager/testing/testing.go b/packager/testing/testing.go index 457d034b9..c15bffc67 100644 --- a/packager/testing/testing.go +++ b/packager/testing/testing.go @@ -29,14 +29,14 @@ import ( // A cert (with its issuer chain) for testing. var Certs = func() []*x509.Certificate { - certPem, _ := ioutil.ReadFile("../../testdata/b1/fullchain.cert") + certPem, _ := ioutil.ReadFile("../../testdata/b3/fullchain.cert") certs, _ := signedexchange.ParseCertificates(certPem) return certs }() // Its corresponding private key. var Key = func() crypto.PrivateKey { - keyPem, _ := ioutil.ReadFile("../../testdata/b1/server.privkey") + keyPem, _ := ioutil.ReadFile("../../testdata/b3/server.privkey") // This call to ParsePrivateKey() is needed by util_test.go. key, _ := util.ParsePrivateKey(keyPem) return key diff --git a/packager/util/config.go b/packager/util/config.go index 1896dfe29..d49fc47c0 100644 --- a/packager/util/config.go +++ b/packager/util/config.go @@ -28,9 +28,19 @@ type Config struct { Port int CertFile string // This must be the full certificate chain. KeyFile string // Just for the first cert, obviously. - OCSPCache string + CSRFile string // Certificate Signing Request. + + // When set, both CertFile and NewCertFile will be read/write. CertFile and + // NewCertFile will be set when both are valid and that once CertFile becomes + // invalid, NewCertFile will replace it (CertFile = NewCertFile) and NewCertFile + // will be set to empty. This will also apply to disk copies as well (which + // we may require to be some sort of shared filesystem, if multiple replicas of + // ammpackager are running). + NewCertFile string // The new full certificate chain replacing the expired one. + OCSPCache string ForwardedRequestHeaders []string - URLSet []URLSet + URLSet []URLSet + ACMEConfig *ACMEConfig } type URLSet struct { @@ -50,6 +60,31 @@ type URLPattern struct { SamePath *bool } +type ACMEConfig struct { + Production *ACMEServerConfig + Development *ACMEServerConfig +} + +type ACMEServerConfig struct { + DiscoURL string // ACME Directory Resource URL + AccountURL string // ACME Account URL. If non-empty, we + // will auto-renew cert via ACME. + EmailAddress string // Email address registered with ACME CA. + + // See: https://letsencrypt.org/docs/challenge-types/ + // For non-wildcard domains, only one of HttpChallengePort, HttpWebRootDir or + // TlsChallengePort needs to be present. + // HttpChallengePort means AmpPackager will respond to HTTP challenges via this port. + // HttpWebRootDir means AmpPackager will deposit challenge token in this directory. + // TlsChallengePort means AmpPackager will respond to TLS challenges via this port. + // For wildcard domains, DnsProvider must be set to one of the support LEGO configs: + // https://go-acme.github.io/lego/dns/ + HttpChallengePort int // ACME HTTP challenge port. + HttpWebRootDir string // ACME HTTP web root directory where challenge token will be deposited. + TlsChallengePort int // ACME TLS challenge port. + DnsProvider string // ACME DNS Provider used for challenge. +} + // TODO(twifkak): Extract default values into a function separate from the one // that does the parsing and validation. This would make signer_test and // validation_test less brittle. diff --git a/packager/util/config_test.go b/packager/util/config_test.go index c49414afd..ba8dfdcfb 100644 --- a/packager/util/config_test.go +++ b/packager/util/config_test.go @@ -29,6 +29,7 @@ func TestMinimalValidConfig(t *testing.T) { config, err := ReadConfig([]byte(` CertFile = "cert.pem" KeyFile = "key.pem" + CSRFile = "file.csr" OCSPCache = "/tmp/ocsp" [[URLSet]] [URLSet.Sign] @@ -39,12 +40,13 @@ func TestMinimalValidConfig(t *testing.T) { Port: 8080, CertFile: "cert.pem", KeyFile: "key.pem", + CSRFile: "file.csr", OCSPCache: "/tmp/ocsp", URLSet: []URLSet{{ Sign: &URLPattern{ - Domain: "example.com", - PathRE: stringPtr(".*"), - QueryRE: stringPtr(""), + Domain: "example.com", + PathRE: stringPtr(".*"), + QueryRE: stringPtr(""), MaxLength: 2000, }, }}, @@ -63,16 +65,16 @@ func TestForwardedRequestHeader(t *testing.T) { `)) require.NoError(t, err) assert.Equal(t, Config{ - Port: 8080, - CertFile: "cert.pem", - KeyFile: "key.pem", - OCSPCache: "/tmp/ocsp", + Port: 8080, + CertFile: "cert.pem", + KeyFile: "key.pem", + OCSPCache: "/tmp/ocsp", ForwardedRequestHeaders: []string{"X-Foo", "X-Bar"}, URLSet: []URLSet{{ Sign: &URLPattern{ - Domain: "example.com", - PathRE: stringPtr(".*"), - QueryRE: stringPtr(""), + Domain: "example.com", + PathRE: stringPtr(".*"), + QueryRE: stringPtr(""), MaxLength: 2000, }, }}, @@ -174,6 +176,97 @@ func TestInvalidQueryRE(t *testing.T) { `))), "QueryRE must be a valid regexp") } +func TestOptionalNewCert(t *testing.T) { + config, err := ReadConfig([]byte(` + CertFile = "cert.pem" + KeyFile = "key.pem" + NewCertFile = "newcert.pem" + OCSPCache = "/tmp/ocsp" + [[URLSet]] + [URLSet.Sign] + Domain = "example.com" + `)) + require.NoError(t, err) + assert.Equal(t, Config{ + Port: 8080, + CertFile: "cert.pem", + KeyFile: "key.pem", + NewCertFile: "newcert.pem", + OCSPCache: "/tmp/ocsp", + URLSet: []URLSet{{ + Sign: &URLPattern{ + Domain: "example.com", + PathRE: stringPtr(".*"), + QueryRE: stringPtr(""), + MaxLength: 2000, + }, + }}, + }, *config) +} + +func TestOptionalACMEConfig(t *testing.T) { + config, err := ReadConfig([]byte(` + CertFile = "cert.pem" + KeyFile = "key.pem" + OCSPCache = "/tmp/ocsp" + [[URLSet]] + [URLSet.Sign] + Domain = "example.com" + [ACMEConfig] + [ACMEConfig.Production] + DiscoURL = "prod.disco.url" + AccountURL = "prod.account.url" + EmailAddress = "prodtest@test.com" + HttpChallengePort = 777 + HttpWebRootDir = "web.root.dir" + TlsChallengePort = 333 + DnsProvider = "gcloud" + [ACMEConfig.Development] + DiscoURL = "dev.disco.url" + AccountURL = "dev.account.url" + EmailAddress = "devtest@test.com" + HttpChallengePort = 888 + HttpWebRootDir = "web.root.dir" + TlsChallengePort = 444 + DnsProvider = "gcloud" + `)) + require.NoError(t, err) + assert.Equal(t, Config{ + Port: 8080, + CertFile: "cert.pem", + KeyFile: "key.pem", + OCSPCache: "/tmp/ocsp", + ACMEConfig: &ACMEConfig{ + Production: &ACMEServerConfig{ + DiscoURL: "prod.disco.url", + AccountURL: "prod.account.url", + EmailAddress: "prodtest@test.com", + HttpChallengePort: 777, + HttpWebRootDir: "web.root.dir", + TlsChallengePort: 333, + DnsProvider: "gcloud", + }, + Development: &ACMEServerConfig{ + DiscoURL: "dev.disco.url", + AccountURL: "dev.account.url", + EmailAddress: "devtest@test.com", + HttpChallengePort: 888, + HttpWebRootDir: "web.root.dir", + TlsChallengePort: 444, + DnsProvider: "gcloud", + }, + }, + URLSet: []URLSet{{ + Sign: &URLPattern{ + Domain: "example.com", + PathRE: stringPtr(".*"), + QueryRE: stringPtr(""), + MaxLength: 2000, + }, + }}, + }, *config) +} + func TestSignMissing(t *testing.T) { msg := errorFrom(ReadConfig([]byte(` CertFile = "cert.pem" diff --git a/packager/util/http.go b/packager/util/http.go index 824cb47ab..b1e31ee1a 100644 --- a/packager/util/http.go +++ b/packager/util/http.go @@ -2,9 +2,11 @@ package util import ( "fmt" - "regexp" "net/http" + "regexp" "strings" + + "github.com/pkg/errors" ) // A comma, as defined in https://tools.ietf.org/html/rfc7230#section-7, with @@ -43,13 +45,13 @@ var ConditionalRequestHeaders = map[string]bool{ // Proxy-Connection should also be deleted, per // https://github.com/WICG/webpackage/pull/339. var legacyHeaders = map[string]bool{ - "Connection": true, - "Keep-Alive": true, + "Connection": true, + "Keep-Alive": true, "Proxy-Authenticate": true, - "Proxy-Connection": true, - "Trailer": true, - "Transfer-Encoding": true, - "Upgrade": true, + "Proxy-Connection": true, + "Trailer": true, + "Transfer-Encoding": true, + "Upgrade": true, } // Via is implicitly forwarded and disallowed to be included in @@ -59,8 +61,8 @@ var legacyHeaders = map[string]bool{ // remove it to mitigate the risk of over-signing. var notForwardedRequestHeader = map[string]bool{ "Proxy-Authorization": true, - "Te": true, - "Via": true, + "Te": true, + "Via": true, } // Remove hop-by-hop headers, per https://tools.ietf.org/html/rfc7230#section-6.1. @@ -91,3 +93,26 @@ func haveInvalidForwardedRequestHeader(h string) string { } return "" } + +// Escapes the input and surrounds it in quotes, so it's a valid quoted-string, +// per https://tools.ietf.org/html/rfc7230#section-3.2.6. Returns error if the +// input contains any chars outside of HTAB / SP / VCHAR +// (https://tools.ietf.org/html/rfc5234#appendix-B.1) and thus isn't even +// quotable. +func QuotedString(input string) (string, error) { + var ret strings.Builder + ret.WriteByte('"') + for i := 0; i < len(input); i++ { + b := input[i] + if (b < 0x20 || b > 0x7e) && b != 0x09 { + return "", errors.New("contains non-printable char") + } + if b == '"' || b == '\\' { + ret.Write([]byte{'\\', b}) + } else { + ret.WriteByte(b) + } + } + ret.WriteByte('"') + return ret.String(), nil +} diff --git a/packager/util/http_test.go b/packager/util/http_test.go new file mode 100644 index 000000000..08718c7ff --- /dev/null +++ b/packager/util/http_test.go @@ -0,0 +1,16 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestQuotedString(t *testing.T) { + valueFrom := func(s string, _ error) string { return s } + errorFrom := func(_ string, err error) error { return err } + + assert.EqualError(t, errorFrom(QuotedString("abc\ndef")), "contains non-printable char") + assert.Equal(t, `"abc"`, valueFrom(QuotedString("abc"))) + assert.Equal(t, `"abc\"\\"`, valueFrom(QuotedString(`abc"\`))) +} diff --git a/packager/util/util.go b/packager/util/util.go index 4c22c0d65..78a5ac4c7 100644 --- a/packager/util/util.go +++ b/packager/util/util.go @@ -31,12 +31,6 @@ import ( const CertURLPrefix = "/amppkg/cert" -// https://wicg.github.io/webpackage/draft-yasskin-http-origin-signed-responses.html#cross-origin-cert-req -// Clients MUST reject certificates with this extension that were issued after 2019-05-01 and have a Validity Period longer than 90 days. -// After 2019-08-01, clients MUST reject all certificates with this extension that have a Validity Period longer than 90 days. -var start90DayGracePeriod = time.Date(2019, time.May, 1, 0, 0, 0, 0, time.UTC) -var end90DayGracePeriod = time.Date(2019, time.August, 1, 0, 0, 0, 0, time.UTC) - // CertName returns the basename for the given cert, as served by this // packager's cert cache. Should be stable and unique (e.g. // content-addressing). Clients should url.PathEscape this, just in case its @@ -47,6 +41,7 @@ func CertName(cert *x509.Certificate) string { } const ValidityMapPath = "/amppkg/validity" +const HealthzPath = "/healthz" // ParsePrivateKey returns the first PEM block that looks like a private key. func ParsePrivateKey(keyPem []byte) (crypto.PrivateKey, error) { @@ -82,20 +77,31 @@ func hasCanSignHttpExchangesExtension(cert *x509.Certificate) bool { return false } +// Returns the Duration of time before cert expires with given deadline. +// Note that the certExpiryDeadline should be the expected SXG expiration time. +// Returns error if cert is already expired. This will be used to periodically check if cert +// is still within validity range. +func GetDurationToExpiry(cert *x509.Certificate, certExpiryDeadline time.Time) (time.Duration, error) { + if cert.NotBefore.After(certExpiryDeadline) { + return 0, errors.New("Certificate is future-dated") + } + if cert.NotAfter.Before(certExpiryDeadline) { + return 0, errors.New("Certificate is expired") + } + + return cert.NotAfter.Sub(certExpiryDeadline), nil +} + // CanSignHttpExchanges returns nil if the given certificate has the // CanSignHttpExchanges extension, and a valid lifetime per the SXG spec; // otherwise it returns an error. These are not the only requirements for SXGs; // it also needs to use the right public key type, which is not checked here. -func CanSignHttpExchanges(cert *x509.Certificate, now time.Time) error { +func CanSignHttpExchanges(cert *x509.Certificate) error { if !hasCanSignHttpExchangesExtension(cert) { return errors.New("Certificate is missing CanSignHttpExchanges extension") } - - // TODO: remove issue date and current time check after 2019-08-01 - if cert.NotBefore.After(start90DayGracePeriod) || now.After(end90DayGracePeriod) { - if cert.NotBefore.AddDate(0,0,90).Before(cert.NotAfter) { - return errors.New("Certificate MUST have a Validity Period no greater than 90 days") - } + if cert.NotBefore.AddDate(0, 0, 90).Before(cert.NotAfter) { + return errors.New("Certificate MUST have a Validity Period no greater than 90 days") } return nil } diff --git a/packager/util/util_test.go b/packager/util/util_test.go index 12f79d356..4133e6cc1 100644 --- a/packager/util/util_test.go +++ b/packager/util/util_test.go @@ -20,23 +20,47 @@ func errorFrom(err error) string { } func TestCertName(t *testing.T) { - assert.Equal(t, "PJ1IwfP1igOlJd2oTUVs2mj4dWIZcOWHMk5jfJYS2Qc", util.CertName(pkgt.Certs[0])) + assert.Equal(t, "Qk83Jo8qB8cEtxfb_7eit0SWVt0pdj5e7oDCqEgf77o", util.CertName(pkgt.B3Certs[0])) } -// ParsePrivateKey() is tested indirectly via the definition of pkgt.Key. +func TestGetDurationToExpiry(t *testing.T) { + // Time before the cert validity. + beforeCert := time.Date(2019, time.May, 8, 0, 0, 0, 0, time.UTC) + // Time after the cert validity. + afterCert := time.Date(2019, time.August, 8, 0, 0, 0, 0, time.UTC) + // Time 2 days before cert validity expiration. + twoDaysBeforeExpiry := time.Date(2019, time.August, 5, 5, 43, 32, 0, time.UTC) + // Time 0 days, 1 hour before cert validity expiration. + oneHourBeforeExpiry := time.Date(2019, time.August, 7, 4, 43, 32, 0, time.UTC) + // Time 0 days before cert validity expiration. + zeroDaysBeforeExpiry := time.Date(2019, time.August, 7, 5, 43, 32, 0, time.UTC) + + d, err := util.GetDurationToExpiry(pkgt.B3Certs[0], beforeCert) + assert.EqualError(t, err, "Certificate is future-dated") + d, err = util.GetDurationToExpiry(pkgt.B3Certs[0], afterCert) + assert.EqualError(t, err, "Certificate is expired") + + d, err = util.GetDurationToExpiry(pkgt.B3Certs[0], twoDaysBeforeExpiry) + assert.Equal(t, time.Duration(2*time.Hour*24), d) + + d, err = util.GetDurationToExpiry(pkgt.B3Certs[0], oneHourBeforeExpiry) + assert.Equal(t, time.Duration(1*time.Hour), d) + + d, err = util.GetDurationToExpiry(pkgt.B3Certs[0], zeroDaysBeforeExpiry) + assert.Equal(t, time.Duration(0), d) +} + +// ParsePrivateKey() is tested indirectly via the definition of pkgt.B3Key. func TestParsePrivateKey(t *testing.T) { - require.IsType(t, &ecdsa.PrivateKey{}, pkgt.Key) - assert.Equal(t, elliptic.P256(), pkgt.Key.(*ecdsa.PrivateKey).PublicKey.Curve) + require.IsType(t, &ecdsa.PrivateKey{}, pkgt.B3Key) + assert.Equal(t, elliptic.P256(), pkgt.B3Key.(*ecdsa.PrivateKey).PublicKey.Curve) } func TestCanSignHttpExchangesExtension(t *testing.T) { - // Before grace period, to allow the >90-day lifetime. - now := time.Date(2019, time.July, 31, 0, 0, 0, 0, time.UTC) - // Leaf node has the extension. - assert.Nil(t, util.CanSignHttpExchanges(pkgt.Certs[0], now)) + assert.Nil(t, util.CanSignHttpExchanges(pkgt.B3Certs[0])) // CA node does not. - assert.EqualError(t, util.CanSignHttpExchanges(pkgt.Certs[1], now), "Certificate is missing CanSignHttpExchanges extension") + assert.EqualError(t, util.CanSignHttpExchanges(pkgt.B3Certs[1]), "Certificate is missing CanSignHttpExchanges extension") } func TestParseCertificate(t *testing.T) { @@ -62,23 +86,7 @@ func TestParseCertificateNotMatchDomain(t *testing.T) { pkgt.B3Key2, "amppackageexample.com")), "x509: certificate is valid for amppackageexample2.com, www.amppackageexample2.com, not amppackageexample.com") } -func TestParse90DaysCertificateAfterGracePeriod(t *testing.T) { - now := time.Date(2019, time.August, 1, 0, 0, 0, 1, time.UTC) - assert.Nil(t, util.CanSignHttpExchanges(pkgt.B3Certs[0], now)) -} - func TestParse91DaysCertificate(t *testing.T) { - assert.Contains(t, errorFrom(util.CanSignHttpExchanges(pkgt.B3Certs91Days[0], - time.Now())), "Certificate MUST have a Validity Period no greater than 90 days") -} - -func TestParseCertificateIssuedBeforeMay1InGarcePeriod(t *testing.T) { - now := time.Date(2019, time.July, 31, 0, 0, 0, 0, time.UTC) - assert.Nil(t, util.CanSignHttpExchanges(pkgt.Certs[0], now)) -} - -func TestParseCertificateIssuedBeforeMay1AfterGracePeriod(t *testing.T) { - now := time.Date(2019, time.August, 1, 0, 0, 0, 1, time.UTC) - assert.Contains(t, errorFrom(util.CanSignHttpExchanges(pkgt.Certs[0], - now)), "Certificate MUST have a Validity Period no greater than 90 days") + assert.Contains(t, errorFrom(util.CanSignHttpExchanges(pkgt.B3Certs91Days[0])), + "Certificate MUST have a Validity Period no greater than 90 days") } diff --git a/packager/validitymap/validitymap_test.go b/packager/validitymap/validitymap_test.go index 4158236b4..b3373b505 100644 --- a/packager/validitymap/validitymap_test.go +++ b/packager/validitymap/validitymap_test.go @@ -14,7 +14,7 @@ func TestValidityMap(t *testing.T) { handler, err := New() require.NoError(t, err) - resp := pkgt.Get(t, mux.New(nil, nil, handler), "/amppkg/validity") + resp := pkgt.Get(t, mux.New(nil, nil, handler, nil), "/amppkg/validity") defer resp.Body.Close() assert.Equal(t, "application/cbor", resp.Header.Get("Content-Type")) assert.Equal(t, "public, max-age=604800", resp.Header.Get("Cache-Control")) diff --git a/testdata/b1/README.md b/testdata/b1/README.md deleted file mode 100644 index 57c290b86..000000000 --- a/testdata/b1/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Test certificates for b1 - -This is an example certificate built under the constraints set by `v=b1` (and -possibly also applicable to higher versions). - -To generate: - -``` -$ openssl genrsa -out ca.privkey 2048 -$ openssl req -x509 -new -nodes -key ca.privkey -sha256 -days 1825 -out ca.cert -subj '/C=US/ST=California/O=Google LLC/CN=Fake CA' -$ openssl ecparam -out server.privkey -name prime256v1 -genkey -$ openssl req -new -sha256 -key server.privkey -out server.csr -subj /CN=example.com -$ openssl x509 -req -in server.csr -CA ca.cert -CAkey ca.privkey -CAcreateserial -out server.cert -days 3650 \ - -extfile <(echo -e "1.3.6.1.4.1.11129.2.1.22 = ASN1:NULL") -$ cat server.cert ca.cert >fullchain.cert -``` - -## OCSP - -To regenerate the OCSP signing certificate: - -``` -$ openssl req -new -sha256 -key ca.privkey -out ocsp.csr -subj '/C=US/ST=California/O=Google LLC/CN=ocsp.example.com' -$ openssl x509 -req -in ocsp.csr -signkey ca.privkey -out ca.ocsp.cert \ - -extfile <(echo -e "keyUsage = critical, digitalSignature\nextendedKeyUsage = critical, OCSPSigning\n") - -``` - -Generate an OCSP response to seed the cache. Note this step must be done -manually on demand because of the 7 day expiry. DO NOT commit the generated -file. - - -``` -$ ./seedcache.sh -``` - - -### Appendix - - - -See some tutorials: - - https://github.com/WICG/webpackage/tree/master/go/signedexchange - - https://deliciousbrains.com/ssl-certificate-authority-for-local-https-development/ - - https://github.com/jmmcatee/cracklord/wiki/Creating-Certificate-Authentication-From-Scratch-OpenSSL - - https://gist.github.com/Soarez/9688998 - - https://jamielinux.com/docs/openssl-certificate-authority/online-certificate-status-protocol.html diff --git a/testdata/b1/blah b/testdata/b1/blah deleted file mode 100644 index 606ef925e..000000000 --- a/testdata/b1/blah +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDaDCCAlCgAwIBAgIJAIStPjotzJxPMA0GCSqGSIb3DQEBCwUAMEkxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRAwDgYDVQQDDAdGYWtlIENBMB4XDTE4MDgyOTIzNTAxN1oXDTIzMDgyODIzNTAx -N1owSTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEzARBgNVBAoM -Ckdvb2dsZSBMTEMxEDAOBgNVBAMMB0Zha2UgQ0EwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQC52bjEZwQIt5pIZY712P6nbQOoRGKmalU6SksHjhVx8x93 -58Fjs+z1S1xm+HwU8LDj82wapj2/GZbLgk6wiYeSF17a4cKbdOfZUqQiQpQfciAB -S3sd2ZG0YghR/VdTHa3mH56lX90Z/3pq0KlwIDk1U/PpKYaWwC1Ywpj1COEJ9Omd -TjcoPQ1nEc1vKdlyjlAvDYqKptrK2grGykvu0Rh2ZeJ99YSSYknh6zx6U3FWtKlm -PgMjSWO2gtpbMLPI3Uehde0b3Bq8JntjPvbh8gLoGaOEsvvDgTfFRpRkNqMAjLVf -z/zol9CpBdggg4Fyj9J0CGjM9f1ZBhXLmtoxkAQjAgMBAAGjUzBRMB0GA1UdDgQW -BBSiY4BoP6E4Rx6tSi0iXDHU5D3vlTAfBgNVHSMEGDAWgBSiY4BoP6E4Rx6tSi0i -XDHU5D3vlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB7Ks29 -9wgWvCneS/82aU9VPfmfCbYDVs95HFYDuXemfbIYoX1GhMuT/wXEEiNnkb1gZXeR -6s6fjc2rLfcUoijSwQdtS9z9D6SXCT3d4h/PsdZoD7+iCQ0BISwrLreW67G8dx9I -b3hAeGupq3iTMdis3z1nHSGALp8wUEoZGl7ZFVCzjUO3/mxnaKXfKLMFUJxe58P/ -qnVSnXygMlhg0zCqPpnaGe1pSe/uC/ax7RK2lLVtrbNW8QMWvoqqJRuAFGf7P+37 -EOCKpQFnRHayhNuftwzAJlM1tDkTx46Lfij8bhfxL8r7Y8wQJvtTc7t1rQF/JsJ8 -i9TINO9JT0tBokim ------END CERTIFICATE----- diff --git a/testdata/b1/ca.cert b/testdata/b1/ca.cert deleted file mode 100644 index 3c25bf9cd..000000000 --- a/testdata/b1/ca.cert +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDaDCCAlCgAwIBAgIJAKRDrJBUupZAMA0GCSqGSIb3DQEBCwUAMEkxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRAwDgYDVQQDDAdGYWtlIENBMB4XDTE4MDgyOTIxMjIyOVoXDTIzMDgyODIxMjIy -OVowSTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEzARBgNVBAoM -Ckdvb2dsZSBMTEMxEDAOBgNVBAMMB0Zha2UgQ0EwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQC52bjEZwQIt5pIZY712P6nbQOoRGKmalU6SksHjhVx8x93 -58Fjs+z1S1xm+HwU8LDj82wapj2/GZbLgk6wiYeSF17a4cKbdOfZUqQiQpQfciAB -S3sd2ZG0YghR/VdTHa3mH56lX90Z/3pq0KlwIDk1U/PpKYaWwC1Ywpj1COEJ9Omd -TjcoPQ1nEc1vKdlyjlAvDYqKptrK2grGykvu0Rh2ZeJ99YSSYknh6zx6U3FWtKlm -PgMjSWO2gtpbMLPI3Uehde0b3Bq8JntjPvbh8gLoGaOEsvvDgTfFRpRkNqMAjLVf -z/zol9CpBdggg4Fyj9J0CGjM9f1ZBhXLmtoxkAQjAgMBAAGjUzBRMB0GA1UdDgQW -BBSiY4BoP6E4Rx6tSi0iXDHU5D3vlTAfBgNVHSMEGDAWgBSiY4BoP6E4Rx6tSi0i -XDHU5D3vlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCG9ARQ -S1tOW0PVuJ5H4WU0hRmDn5oUkEu9DzD1WoGGTN6CCM0AfLBo0ARwqntvSNtERIA6 -BfVSaeqO6WNrvGDLwi+YcUlTxyGLsGlDz/xeQ7WIVGlbhyTJYH5B1yliRyZN4M2E -hIqWqBMSvtqCnMMJEr/0BsigDIEiEUXwsc6tC77HudXxV0AVs4MLBz3iowkeZMB6 -qZAVRKCmVptIsPyP+eO3vIIWi0y7eQx/8WOpW52CYj9MhhDa9/IcJLIRQIOe7fRn -1tL/h+aRMq7ywOrwbO/s2HQ7kG/jFIRj0QMxeCExTzPhND22EVvVisfocXS91QuL -PjZuIWmKwDo/Vvau ------END CERTIFICATE----- diff --git a/testdata/b1/ca.ocsp.cert b/testdata/b1/ca.ocsp.cert deleted file mode 100644 index 69965a966..000000000 --- a/testdata/b1/ca.ocsp.cert +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDUTCCAjmgAwIBAgIJALoE1bsaMrFKMA0GCSqGSIb3DQEBCwUAMFIxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRkwFwYDVQQDDBBvY3NwLmV4YW1wbGUuY29tMB4XDTE4MTEwNTE5MzIxMloXDTE4 -MTIwNTE5MzIxMlowUjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx -EzARBgNVBAoMCkdvb2dsZSBMTEMxGTAXBgNVBAMMEG9jc3AuZXhhbXBsZS5jb20w -ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC52bjEZwQIt5pIZY712P6n -bQOoRGKmalU6SksHjhVx8x9358Fjs+z1S1xm+HwU8LDj82wapj2/GZbLgk6wiYeS -F17a4cKbdOfZUqQiQpQfciABS3sd2ZG0YghR/VdTHa3mH56lX90Z/3pq0KlwIDk1 -U/PpKYaWwC1Ywpj1COEJ9OmdTjcoPQ1nEc1vKdlyjlAvDYqKptrK2grGykvu0Rh2 -ZeJ99YSSYknh6zx6U3FWtKlmPgMjSWO2gtpbMLPI3Uehde0b3Bq8JntjPvbh8gLo -GaOEsvvDgTfFRpRkNqMAjLVfz/zol9CpBdggg4Fyj9J0CGjM9f1ZBhXLmtoxkAQj -AgMBAAGjKjAoMA4GA1UdDwEB/wQEAwIHgDAWBgNVHSUBAf8EDDAKBggrBgEFBQcD -CTANBgkqhkiG9w0BAQsFAAOCAQEAStm0n/tspDf5ooVy8VoQslMdeMpBDe7VPhzp -Fl9HvTKtxxSohtt7BhGN7HzSgW5bAqeGUxiQ7aiizv5NJv348j+aI/+cX/nOo+VG -n60NAkkmKA8MW1e04kLWr3JL+qTejv9HXM3aeik80lyjqOKd5ZGxHFsdObto7I+U -Y6EC3r300/sVSgluYrSr3qpS2OwdY0D/8mfp7H3gXQQfNT3d0tSq4lBCGQfz8Gwe -tpEaPQWptq0nO5U5tZqMNmZluHcUxaMMd99kCgDCDQkSQO3TSqEP4guO73h8TYoM -fIV5BaJd560e1noyYxpFuiN35bUC3m49i5WcG7ZxZodoYclYRA== ------END CERTIFICATE----- diff --git a/testdata/b1/ca.privkey b/testdata/b1/ca.privkey deleted file mode 100644 index a3f2c488e..000000000 --- a/testdata/b1/ca.privkey +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAudm4xGcECLeaSGWO9dj+p20DqERipmpVOkpLB44VcfMfd+fB -Y7Ps9UtcZvh8FPCw4/NsGqY9vxmWy4JOsImHkhde2uHCm3Tn2VKkIkKUH3IgAUt7 -HdmRtGIIUf1XUx2t5h+epV/dGf96atCpcCA5NVPz6SmGlsAtWMKY9QjhCfTpnU43 -KD0NZxHNbynZco5QLw2KiqbaytoKxspL7tEYdmXiffWEkmJJ4es8elNxVrSpZj4D -I0ljtoLaWzCzyN1HoXXtG9wavCZ7Yz724fIC6BmjhLL7w4E3xUaUZDajAIy1X8/8 -6JfQqQXYIIOBco/SdAhozPX9WQYVy5raMZAEIwIDAQABAoIBACHsv1CCqXbZ5PzQ -JQ91g86WFLPTf9p20IXqZ9XCNuHtClJ96IxFnLyN/BkDxMqhwPhrR9F5hQ3sIt2V -NL3+7NNbFsKHsVllNqkx76odUyKGV5dE6v1g6LrvpispPpZ6dXLrVK9FV3vWaccz -vaotB6RXZc+q99luzRhFtVwNOd7yGQWkn1sKO1i/QWiARdob/inKArbNl3W0GtJ0 -kIjQNm28JFpzTxUbjoWOOUoH4B8PyOS+k6+ByjJDBhLiinl3I7M+g6LWqr2cdam8 -xyoHHyUtYM1DbCoPJKDtJeJkxNP9TZTJLLq/lqRjm14kvcIHvQxFSZF/NqJivurq -LYtghDkCgYEA7SC1ot2PjkZ2MZNQQsT5HrAwqqfogb9AvQ2HSkFN71LGZQcfxhNS -chg1/MH3R9k95xDPSFc2T5DxEBUFaImeGhPNnm1YzHJbixZgBu2G6DAEks95qtYf -bDWRFG60ulaJAeveVYxhnoXARc9NMPAbek6VgFi1QWFY+M/VTC3cve8CgYEAyKRH -J6eUurQZti9mUwbSPWB41+hi7KFL3qC618qOf0XDnDlGvhSxkMLdnICMDOxg6U/b -0G4Zlyb6JbyZ5jiDpvcAbqqGsickmI8SJeKbp9zQT0mELuRdx12YuMi0aA81gaIm -ekYDiQp28enO778s033ikrYwVu2yLakrFayakQ0CgYEAnc8Z8nSfGCF+gUm3rWfn -HuxExx4Nl2OPkwGQ2vMRCce9rviJxcmQIcxJCZiQl+lU0BUYzdz0kQk11O0Yd1S2 -ukYZnmjJIu6sS6ktaQ7krFtgf8/B+dacfOg9UCrI7gWvEm9FvQs64EPFDPCEP6Bb -uQ7ZYdwnbIZ7rsKqAhO3h1MCgYBfGwejc1sbmO0rH5K4Pl5/u2/sn/nsQpStBbEr -QpeDGrWbIsc2qKZ2gPf9DC3WnmFdln4ScW3t6QrfwmOM7jLxfNmWm3xXjBhbvE2U -6bJwwkl3m9htRdByBRq0VGa3gKYTOaJViUR5vB0flH2DxTHhWiWA9504R1mTLUH/ -9x4ZLQKBgE039cFcqazKgUrhGfnHo6DKMYv9tLfpwBrfGrjS5eL9HzwCjqflH7Ql -rN/8D4hNeT+lU9m8gAZkf7a/9Atz+V97gIKRzLg8ShHPg4lcjHaAK35CRSrNc+i0 -Hf1WpUKryO9CCzqsmO1Z+hxLVC1rSYJ2C8TRI3FMf3Gay/IzUUXF ------END RSA PRIVATE KEY----- diff --git a/testdata/b1/ca.srl b/testdata/b1/ca.srl deleted file mode 100644 index 588ec9b34..000000000 --- a/testdata/b1/ca.srl +++ /dev/null @@ -1 +0,0 @@ -FA2006BB9AAF740D diff --git a/testdata/b1/fullchain.cert b/testdata/b1/fullchain.cert deleted file mode 100644 index c04da576f..000000000 --- a/testdata/b1/fullchain.cert +++ /dev/null @@ -1,35 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNTCCAR2gAwIBAgIJAPogBruar3QNMA0GCSqGSIb3DQEBCwUAMEkxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRAwDgYDVQQDDAdGYWtlIENBMB4XDTE4MTEwMjIzMzAyMloXDTI4MTAzMDIzMzAy -MlowIDEeMBwGA1UEAwwVYW1wcGFja2FnZWV4YW1wbGUuY29tMFkwEwYHKoZIzj0C -AQYIKoZIzj0DAQcDQgAEpq4N0s1Ye7mBanP8EZXizWZi3Dsb6vQlVIv7CI19yVuo -wmqK2a5ADyKPyoZ6KbVfY/V5YSKhBVQJT1PRp5d0jqMUMBIwEAYKKwYBBAHWeQIB -FgQCBQAwDQYJKoZIhvcNAQELBQADggEBAKgLPVYS8dlg4NnDmeePMNelX0aoONV2 -pijVBjVkFf8lN00TVgthNDPZlk9Vxq7dxnU6ATbsb62JbZRoSPRxW7dk91YOb3ht -12tzTlFbdX485k+KsREdCFbu9cpF8I34VGKcZUsi7u7uMZFfi8S+XcMyC1EQpueQ -+xlOTUQAOXspg+qQ1gwlOSznV0UZ+4B2/XkghOkweZfwla/62WzgtmD8a9uuoVd0 -RARwfB/D7ye9mvqbMyiOc3uO5zfCUrtiW5oin4SEgTGYvuEcHRaMV6cjx1r7MBLt -KpIgRXTPWLAVo9OLxmDKUZH23DYVAx8BIBDpQzoqchmgW1X9eX1k1Y8= ------END CERTIFICATE----- ------BEGIN CERTIFICATE----- -MIIDaDCCAlCgAwIBAgIJAKRDrJBUupZAMA0GCSqGSIb3DQEBCwUAMEkxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRAwDgYDVQQDDAdGYWtlIENBMB4XDTE4MDgyOTIxMjIyOVoXDTIzMDgyODIxMjIy -OVowSTELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExEzARBgNVBAoM -Ckdvb2dsZSBMTEMxEDAOBgNVBAMMB0Zha2UgQ0EwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQC52bjEZwQIt5pIZY712P6nbQOoRGKmalU6SksHjhVx8x93 -58Fjs+z1S1xm+HwU8LDj82wapj2/GZbLgk6wiYeSF17a4cKbdOfZUqQiQpQfciAB -S3sd2ZG0YghR/VdTHa3mH56lX90Z/3pq0KlwIDk1U/PpKYaWwC1Ywpj1COEJ9Omd -TjcoPQ1nEc1vKdlyjlAvDYqKptrK2grGykvu0Rh2ZeJ99YSSYknh6zx6U3FWtKlm -PgMjSWO2gtpbMLPI3Uehde0b3Bq8JntjPvbh8gLoGaOEsvvDgTfFRpRkNqMAjLVf -z/zol9CpBdggg4Fyj9J0CGjM9f1ZBhXLmtoxkAQjAgMBAAGjUzBRMB0GA1UdDgQW -BBSiY4BoP6E4Rx6tSi0iXDHU5D3vlTAfBgNVHSMEGDAWgBSiY4BoP6E4Rx6tSi0i -XDHU5D3vlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCG9ARQ -S1tOW0PVuJ5H4WU0hRmDn5oUkEu9DzD1WoGGTN6CCM0AfLBo0ARwqntvSNtERIA6 -BfVSaeqO6WNrvGDLwi+YcUlTxyGLsGlDz/xeQ7WIVGlbhyTJYH5B1yliRyZN4M2E -hIqWqBMSvtqCnMMJEr/0BsigDIEiEUXwsc6tC77HudXxV0AVs4MLBz3iowkeZMB6 -qZAVRKCmVptIsPyP+eO3vIIWi0y7eQx/8WOpW52CYj9MhhDa9/IcJLIRQIOe7fRn -1tL/h+aRMq7ywOrwbO/s2HQ7kG/jFIRj0QMxeCExTzPhND22EVvVisfocXS91QuL -PjZuIWmKwDo/Vvau ------END CERTIFICATE----- diff --git a/testdata/b1/seedcache.sh b/testdata/b1/seedcache.sh deleted file mode 100755 index 702405985..000000000 --- a/testdata/b1/seedcache.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Seed the OCSP cache using the fake test certs. Do NOT run this if you -# are using real certificates with the AMP Packager. - -openssl ocsp -index ./index.txt -rsigner ca.ocsp.cert -rkey ca.privkey -CA ca.cert -ndays 7 -issuer ca.cert -cert server.cert -respout /tmp/amppkg-ocsp diff --git a/testdata/b1/server.cert b/testdata/b1/server.cert deleted file mode 100644 index efed40061..000000000 --- a/testdata/b1/server.cert +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICNTCCAR2gAwIBAgIJAPogBruar3QNMA0GCSqGSIb3DQEBCwUAMEkxCzAJBgNV -BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRMwEQYDVQQKDApHb29nbGUgTExD -MRAwDgYDVQQDDAdGYWtlIENBMB4XDTE4MTEwMjIzMzAyMloXDTI4MTAzMDIzMzAy -MlowIDEeMBwGA1UEAwwVYW1wcGFja2FnZWV4YW1wbGUuY29tMFkwEwYHKoZIzj0C -AQYIKoZIzj0DAQcDQgAEpq4N0s1Ye7mBanP8EZXizWZi3Dsb6vQlVIv7CI19yVuo -wmqK2a5ADyKPyoZ6KbVfY/V5YSKhBVQJT1PRp5d0jqMUMBIwEAYKKwYBBAHWeQIB -FgQCBQAwDQYJKoZIhvcNAQELBQADggEBAKgLPVYS8dlg4NnDmeePMNelX0aoONV2 -pijVBjVkFf8lN00TVgthNDPZlk9Vxq7dxnU6ATbsb62JbZRoSPRxW7dk91YOb3ht -12tzTlFbdX485k+KsREdCFbu9cpF8I34VGKcZUsi7u7uMZFfi8S+XcMyC1EQpueQ -+xlOTUQAOXspg+qQ1gwlOSznV0UZ+4B2/XkghOkweZfwla/62WzgtmD8a9uuoVd0 -RARwfB/D7ye9mvqbMyiOc3uO5zfCUrtiW5oin4SEgTGYvuEcHRaMV6cjx1r7MBLt -KpIgRXTPWLAVo9OLxmDKUZH23DYVAx8BIBDpQzoqchmgW1X9eX1k1Y8= ------END CERTIFICATE----- diff --git a/testdata/b1/server.csr b/testdata/b1/server.csr deleted file mode 100644 index d624da951..000000000 --- a/testdata/b1/server.csr +++ /dev/null @@ -1,7 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIHbMIGCAgEAMCAxHjAcBgNVBAMMFWFtcHBhY2thZ2VleGFtcGxlLmNvbTBZMBMG -ByqGSM49AgEGCCqGSM49AwEHA0IABKauDdLNWHu5gWpz/BGV4s1mYtw7G+r0JVSL -+wiNfclbqMJqitmuQA8ij8qGeim1X2P1eWEioQVUCU9T0aeXdI6gADAKBggqhkjO -PQQDAgNIADBFAiEAhwIm8LIS+pB8/2GBqRX81R0n7Hyl0aJswhEP/bvK0x8CIB8U -Idu3yFpZLOo1YMUWGTNyatzHYh4BwuTDquL0dcPA ------END CERTIFICATE REQUEST----- diff --git a/testdata/b1/server.privkey b/testdata/b1/server.privkey deleted file mode 100644 index b2425fecb..000000000 --- a/testdata/b1/server.privkey +++ /dev/null @@ -1,8 +0,0 @@ ------BEGIN EC PARAMETERS----- -BggqhkjOPQMBBw== ------END EC PARAMETERS----- ------BEGIN EC PRIVATE KEY----- -MHcCAQEEIBFEv3eTdzTWBkVImcDme29EoK5bfPyZ6aD/gDiwNUmkoAoGCCqGSM49 -AwEHoUQDQgAEpq4N0s1Ye7mBanP8EZXizWZi3Dsb6vQlVIv7CI19yVuowmqK2a5A -DyKPyoZ6KbVfY/V5YSKhBVQJT1PRp5d0jg== ------END EC PRIVATE KEY----- diff --git a/transformer/README.md b/transformer/README.md index a464044a3..bd3fa6c34 100644 --- a/transformer/README.md +++ b/transformer/README.md @@ -3,10 +3,8 @@ The modifications in this package are described in more detail [here](https://github.com/ampproject/amphtml/blob/master/spec/amp-cache-modifications.md). -> NOTE: The transformed AMP HTML produced by the library is only valid inside of -> a signed exchange, and not to be served as normal HTML. Also, the library is -> still a work-in-progress and not all transformations described in the link -> above are implemented. +The transformed AMP HTML produced by the library is meant to be used inside of +a signed exchange, but may be valid in other contexts, as well. ## How to use The local transformer can be used separately from the packager/signer. Here's an diff --git a/transformer/internal/htmlnode/htmlnode.go b/transformer/internal/htmlnode/htmlnode.go index 069d5f33f..99115708e 100644 --- a/transformer/internal/htmlnode/htmlnode.go +++ b/transformer/internal/htmlnode/htmlnode.go @@ -131,6 +131,18 @@ func HasAttribute(n *html.Node, namespace, key string) bool { return ok } +// HasAttributeAndIsNotEmpty return true if the node has the attribute named +// with 'key' and it's value is not empty. +func HasAttributeAndIsNotEmpty(n *html.Node, namespace, key string) bool { + if v, ok := GetAttributeVal(n, namespace, key); ok { + if v != "" { + return true + } + return false + } + return false +} + // SetAttribute overrides the value of the attribute on node n with // namespace and key with val. If the attribute doesn't exist, it adds it. func SetAttribute(n *html.Node, namespace, key, val string) { diff --git a/transformer/layout/layout.go b/transformer/layout/layout.go index c7487c1ae..938fabb4f 100644 --- a/transformer/layout/layout.go +++ b/transformer/layout/layout.go @@ -86,8 +86,8 @@ func ApplyLayout(n *html.Node) error { } actualLayout, err := getNormalizedLayout( inputLayout, dimensions, - htmlnode.GetAttributeValOrNil(n, "", "sizes"), - htmlnode.GetAttributeValOrNil(n, "", "heights")) + htmlnode.HasAttributeAndIsNotEmpty(n, "", "sizes"), + htmlnode.HasAttributeAndIsNotEmpty(n, "", "heights")) if err != nil { return err } @@ -143,7 +143,7 @@ func getNormalizedDimensions(n *html.Node, layout amppb.AmpLayout_Layout) (cssDi // getNormalizedLayout returns the normalized AmpLayout based on the // provided dimensions, or an error if it is not supported. -func getNormalizedLayout(layout amppb.AmpLayout_Layout, dimensions cssDimensions, sizes, heights *string) (amppb.AmpLayout_Layout, error) { +func getNormalizedLayout(layout amppb.AmpLayout_Layout, dimensions cssDimensions, sizes, heights bool) (amppb.AmpLayout_Layout, error) { var result amppb.AmpLayout_Layout if layout != amppb.AmpLayout_UNKNOWN { result = layout @@ -153,7 +153,7 @@ func getNormalizedLayout(layout amppb.AmpLayout_Layout, dimensions cssDimensions result = amppb.AmpLayout_FLUID } else if dimensions.height.isSet && (!dimensions.width.isSet || dimensions.width.isAuto) { result = amppb.AmpLayout_FIXED_HEIGHT - } else if dimensions.height.isSet && dimensions.width.isSet && (sizes != nil || heights != nil) { + } else if dimensions.height.isSet && dimensions.width.isSet && (sizes || heights) { result = amppb.AmpLayout_RESPONSIVE } else { result = amppb.AmpLayout_FIXED diff --git a/transformer/layout/layout_test.go b/transformer/layout/layout_test.go index 8fdba28fe..8d52e988b 100644 --- a/transformer/layout/layout_test.go +++ b/transformer/layout/layout_test.go @@ -114,6 +114,20 @@ func TestApplyLayout(t *testing.T) { html.Attribute{Key: "height", Val: "100"}, html.Attribute{Key: "layout", Val: "fixed-height"}), ``, }, + { + "Fixed when heights is empty", + htmlnode.Element( + "amp-img", + html.Attribute{Key: "height", Val: "100"}, html.Attribute{Key: "heights", Val: ""}, html.Attribute{Key: "width", Val: "300"}), + ``, + }, + { + "Fixed when sizes is empty", + htmlnode.Element( + "amp-img", + html.Attribute{Key: "height", Val: "100"}, html.Attribute{Key: "sizes", Val: ""}, html.Attribute{Key: "width", Val: "300"}), + ``, + }, { "Responsive", htmlnode.Element( diff --git a/transformer/request/request.pb.go b/transformer/request/request.pb.go index a521816e1..15ad9975d 100644 --- a/transformer/request/request.pb.go +++ b/transformer/request/request.pb.go @@ -278,10 +278,15 @@ type Metadata struct { // `Link: rel=preload` headers, as these are used by the browser during SXG // prefetch: // https://github.com/WICG/webpackage/blob/master/explainer.md#prefetching-stops-here - Preloads []*Metadata_Preload `protobuf:"bytes,1,rep,name=preloads,proto3" json:"preloads,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Preloads []*Metadata_Preload `protobuf:"bytes,1,rep,name=preloads,proto3" json:"preloads,omitempty"` + // Recommended validity duration (`expires - date`), in seconds, of the SXG, + // based on the content being signed. In particular, JS is given a shorter + // lifetime to reduce risk of issues due to downgrades: + // https://wicg.github.io/webpackage/draft-yasskin-http-origin-signed-responses.html#seccons-downgrades. + MaxAgeSecs int32 `protobuf:"varint,2,opt,name=max_age_secs,json=maxAgeSecs,proto3" json:"max_age_secs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Metadata) Reset() { *m = Metadata{} } @@ -316,6 +321,13 @@ func (m *Metadata) GetPreloads() []*Metadata_Preload { return nil } +func (m *Metadata) GetMaxAgeSecs() int32 { + if m != nil { + return m.MaxAgeSecs + } + return 0 +} + type Metadata_Preload struct { // The URL of the resource to preload. Will be an absolute URL on the domain // of the target AMP cache. @@ -325,7 +337,7 @@ type Metadata_Preload struct { // https://html.spec.whatwg.org/multipage/semantics.html#attr-link-as. The // full list of potential values is specified in // https://fetch.spec.whatwg.org/#concept-request-destination, though for - // the time being only "script", "style" and image are allowed. + // the time being only "script", "style", and "image" are allowed. As string `protobuf:"bytes,2,opt,name=as,proto3" json:"as,omitempty"` // The media attribute for image preload link. This attribute is useful // only for image links. @@ -393,36 +405,38 @@ func init() { func init() { proto.RegisterFile("transformer/request/request.proto", fileDescriptor_762cce2ac5f73405) } var fileDescriptor_762cce2ac5f73405 = []byte{ - // 495 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x52, 0x4d, 0x73, 0xd3, 0x30, - 0x10, 0xad, 0xa3, 0x34, 0x4e, 0x37, 0x69, 0xd0, 0x68, 0x38, 0x68, 0xb8, 0xe0, 0xf8, 0x64, 0x2e, - 0xce, 0x4c, 0x80, 0xe1, 0xc0, 0xc9, 0x24, 0x2e, 0x04, 0x62, 0x27, 0xe3, 0x26, 0x85, 0xe1, 0x92, - 0x51, 0x1d, 0x35, 0x0d, 0xf8, 0x23, 0x48, 0x4a, 0xe9, 0x6f, 0xe0, 0xdf, 0xf1, 0x8f, 0x18, 0xc9, - 0x4e, 0x4b, 0xf9, 0x38, 0x69, 0xf7, 0xe9, 0x3d, 0xe9, 0xcd, 0xee, 0x83, 0xbe, 0x12, 0xac, 0x90, - 0x57, 0xa5, 0xc8, 0xb9, 0x18, 0x08, 0xfe, 0x6d, 0xcf, 0xa5, 0x3a, 0x9c, 0xfe, 0x4e, 0x94, 0xaa, - 0x24, 0xa7, 0x2c, 0xdf, 0xf9, 0x77, 0x34, 0xf7, 0x27, 0x02, 0x3b, 0xa9, 0x08, 0x84, 0x40, 0xf3, - 0x5a, 0xe5, 0x19, 0xb5, 0x1c, 0xcb, 0x3b, 0x49, 0x4c, 0x4d, 0xfa, 0xd0, 0x5d, 0x97, 0xe9, 0x3e, - 0xe7, 0x85, 0x5a, 0xed, 0x45, 0x46, 0x1b, 0xe6, 0xae, 0x73, 0xc0, 0x96, 0x22, 0x23, 0x18, 0x90, - 0x50, 0x37, 0xb4, 0x69, 0x6e, 0x74, 0xa9, 0x91, 0x54, 0x4a, 0x7a, 0x5c, 0x21, 0xa9, 0x94, 0xe4, - 0x3d, 0x3c, 0x62, 0x59, 0x56, 0x7e, 0xe7, 0xeb, 0x95, 0xfe, 0x96, 0x29, 0x49, 0x6d, 0x07, 0x79, - 0xbd, 0x61, 0xdf, 0x7f, 0xe0, 0xc7, 0xaf, 0xbd, 0xf8, 0xef, 0x54, 0x9e, 0x9d, 0x19, 0x66, 0xd2, - 0xab, 0x95, 0x55, 0x2b, 0x49, 0x00, 0xad, 0xb4, 0x2c, 0xae, 0xb6, 0x1b, 0xda, 0x72, 0x2c, 0xaf, - 0x37, 0x7c, 0xf6, 0x9f, 0x27, 0x16, 0xf7, 0xb3, 0x90, 0x23, 0x23, 0x48, 0x6a, 0x21, 0x71, 0xa1, - 0xfb, 0xdb, 0xa4, 0x24, 0x45, 0x0e, 0xf2, 0x4e, 0x92, 0x07, 0x18, 0xa1, 0x60, 0xdf, 0x70, 0x21, - 0xb7, 0x65, 0x41, 0xdb, 0x8e, 0xe5, 0xa1, 0xe4, 0xd0, 0xba, 0x4b, 0x80, 0x7b, 0x7b, 0x04, 0x43, - 0x77, 0x19, 0x7f, 0x88, 0x67, 0x1f, 0xe3, 0xd5, 0x68, 0x36, 0x0e, 0xf1, 0x11, 0xb1, 0x01, 0x05, - 0xd1, 0x1c, 0x5b, 0xa4, 0x03, 0x76, 0x10, 0xcd, 0x5f, 0x04, 0xe3, 0x73, 0xdc, 0x20, 0xa7, 0x70, - 0xa2, 0x9b, 0x30, 0x0a, 0x26, 0x53, 0x8c, 0xb4, 0x2c, 0xfc, 0x34, 0x0f, 0x93, 0x49, 0x14, 0xc6, - 0x8b, 0x60, 0x8a, 0x9b, 0xee, 0x5b, 0x20, 0x7f, 0x5b, 0xd6, 0x6f, 0x8c, 0xc3, 0xb3, 0x60, 0x39, - 0x5d, 0xe0, 0x23, 0xd2, 0x86, 0x66, 0x3c, 0x8b, 0x43, 0x6c, 0x91, 0x1e, 0xc0, 0x45, 0x30, 0x9d, - 0x8c, 0x83, 0xc5, 0x64, 0x16, 0xe3, 0x06, 0x01, 0x68, 0x8d, 0x96, 0xe7, 0x8b, 0x59, 0x84, 0x91, - 0x3b, 0x84, 0xee, 0x45, 0x65, 0x35, 0x61, 0xc5, 0x86, 0xeb, 0x75, 0xe4, 0xdb, 0xc2, 0xac, 0x15, - 0x25, 0xba, 0x34, 0x08, 0xbb, 0x35, 0xcb, 0xd4, 0x08, 0xbb, 0x75, 0x7f, 0x58, 0xd0, 0x8e, 0xb8, - 0x62, 0x6b, 0xa6, 0x18, 0x79, 0x0d, 0xed, 0x9d, 0xe0, 0x59, 0xc9, 0xd6, 0x92, 0x5a, 0x0e, 0xf2, - 0x3a, 0xc3, 0xa7, 0x7f, 0xcc, 0xf8, 0x40, 0xf5, 0xe7, 0x15, 0x2f, 0xb9, 0x13, 0x3c, 0x09, 0xc0, - 0xae, 0x41, 0xfd, 0x8d, 0xce, 0x4c, 0x95, 0x27, 0x5d, 0x92, 0x1e, 0x34, 0x98, 0xac, 0x43, 0xd4, - 0x60, 0x92, 0x3c, 0x86, 0xe3, 0x9c, 0xaf, 0xb7, 0x8c, 0x22, 0x03, 0x55, 0xcd, 0x9b, 0x57, 0x9f, - 0x5f, 0x6e, 0xb6, 0xea, 0x7a, 0x7f, 0xe9, 0xa7, 0x65, 0x3e, 0x60, 0xf9, 0x6e, 0x27, 0xca, 0x2f, - 0x3c, 0x55, 0xa6, 0x64, 0xe9, 0x57, 0xb6, 0xe1, 0x62, 0xf0, 0x8f, 0xa8, 0x5f, 0xb6, 0x4c, 0xc6, - 0x9f, 0xff, 0x0a, 0x00, 0x00, 0xff, 0xff, 0xce, 0x78, 0xb7, 0x04, 0x08, 0x03, 0x00, 0x00, + // 519 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x53, 0x5d, 0x93, 0xd2, 0x30, + 0x14, 0xdd, 0x12, 0xa0, 0x70, 0x61, 0xb1, 0x93, 0xf1, 0xa1, 0xe3, 0x8b, 0xa5, 0x4f, 0xf5, 0xa5, + 0xcc, 0xa0, 0x8e, 0x0f, 0x3e, 0x55, 0xe8, 0x2a, 0x4a, 0x0b, 0x53, 0x60, 0x75, 0x7c, 0x61, 0xb2, + 0x25, 0xdb, 0x45, 0x9b, 0x16, 0x93, 0xb0, 0xf2, 0xa3, 0xfc, 0x33, 0xfe, 0x23, 0x27, 0x29, 0xec, + 0xba, 0x7e, 0x3c, 0xf5, 0xdc, 0xd3, 0x73, 0x92, 0x33, 0xf7, 0xde, 0x40, 0x5f, 0x72, 0x52, 0x88, + 0xeb, 0x92, 0x33, 0xca, 0x07, 0x9c, 0x7e, 0xdb, 0x53, 0x21, 0x4f, 0x5f, 0x7f, 0xc7, 0x4b, 0x59, + 0xe2, 0x73, 0xc2, 0x76, 0xfe, 0x9d, 0xcc, 0xfd, 0x89, 0xc0, 0x4c, 0x2a, 0x01, 0xc6, 0x50, 0xbf, + 0x91, 0x2c, 0xb7, 0x0d, 0xc7, 0xf0, 0xda, 0x89, 0xc6, 0xb8, 0x0f, 0xdd, 0x4d, 0x99, 0xee, 0x19, + 0x2d, 0xe4, 0x7a, 0xcf, 0x73, 0xbb, 0xa6, 0xff, 0x75, 0x4e, 0xdc, 0x8a, 0xe7, 0xd8, 0x02, 0xc4, + 0xe5, 0xad, 0x5d, 0xd7, 0x7f, 0x14, 0x54, 0x4c, 0x2a, 0x84, 0xdd, 0xa8, 0x98, 0x54, 0x08, 0xfc, + 0x1e, 0x1e, 0x91, 0x3c, 0x2f, 0xbf, 0xd3, 0xcd, 0x5a, 0x5d, 0x4b, 0xa4, 0xb0, 0x4d, 0x07, 0x79, + 0xbd, 0x61, 0xdf, 0x7f, 0x90, 0xc7, 0x3f, 0x66, 0xf1, 0xdf, 0x49, 0x96, 0x5f, 0x68, 0x65, 0xd2, + 0x3b, 0x3a, 0xab, 0x52, 0xe0, 0x00, 0x9a, 0x69, 0x59, 0x5c, 0x6f, 0x33, 0xbb, 0xe9, 0x18, 0x5e, + 0x6f, 0xf8, 0xec, 0x3f, 0x47, 0x2c, 0xef, 0x7b, 0x21, 0x46, 0xda, 0x90, 0x1c, 0x8d, 0xd8, 0x85, + 0xee, 0x6f, 0x9d, 0x12, 0x36, 0x72, 0x90, 0xd7, 0x4e, 0x1e, 0x70, 0xd8, 0x06, 0xf3, 0x96, 0x72, + 0xb1, 0x2d, 0x0b, 0xbb, 0xe5, 0x18, 0x1e, 0x4a, 0x4e, 0xa5, 0xbb, 0x02, 0xb8, 0x8f, 0x87, 0x2d, + 0xe8, 0xae, 0xe2, 0x0f, 0xf1, 0xec, 0x63, 0xbc, 0x1e, 0xcd, 0xc6, 0xa1, 0x75, 0x86, 0x4d, 0x40, + 0x41, 0x34, 0xb7, 0x0c, 0xdc, 0x01, 0x33, 0x88, 0xe6, 0x2f, 0x82, 0xf1, 0xc2, 0xaa, 0xe1, 0x73, + 0x68, 0xab, 0x22, 0x8c, 0x82, 0xc9, 0xd4, 0x42, 0xca, 0x16, 0x7e, 0x9a, 0x87, 0xc9, 0x24, 0x0a, + 0xe3, 0x65, 0x30, 0xb5, 0xea, 0xee, 0x5b, 0xc0, 0x7f, 0x47, 0x56, 0x67, 0x8c, 0xc3, 0x8b, 0x60, + 0x35, 0x5d, 0x5a, 0x67, 0xb8, 0x05, 0xf5, 0x78, 0x16, 0x87, 0x96, 0x81, 0x7b, 0x00, 0x97, 0xc1, + 0x74, 0x32, 0x0e, 0x96, 0x93, 0x59, 0x6c, 0xd5, 0x30, 0x40, 0x73, 0xb4, 0x5a, 0x2c, 0x67, 0x91, + 0x85, 0xdc, 0x21, 0x74, 0x2f, 0xab, 0xa8, 0x09, 0x29, 0x32, 0xaa, 0xc6, 0xc1, 0xb6, 0x85, 0x1e, + 0x2b, 0x4a, 0x14, 0xd4, 0x0c, 0x39, 0xe8, 0x61, 0x2a, 0x86, 0x1c, 0xdc, 0x1f, 0x06, 0xb4, 0x22, + 0x2a, 0xc9, 0x86, 0x48, 0x82, 0x5f, 0x43, 0x6b, 0xc7, 0x69, 0x5e, 0x92, 0x8d, 0xb0, 0x0d, 0x07, + 0x79, 0x9d, 0xe1, 0xd3, 0x3f, 0x7a, 0x7c, 0x92, 0xfa, 0xf3, 0x4a, 0x97, 0xdc, 0x19, 0xb0, 0x03, + 0x5d, 0x46, 0x0e, 0x6b, 0x92, 0xd1, 0xb5, 0xa0, 0xa9, 0xd0, 0x97, 0x34, 0x12, 0x60, 0xe4, 0x10, + 0x64, 0x74, 0x41, 0x53, 0xf1, 0x24, 0x00, 0xf3, 0x68, 0x53, 0x41, 0xd4, 0x56, 0x55, 0x1b, 0xa7, + 0x20, 0xee, 0x41, 0x8d, 0x88, 0xe3, 0x9a, 0xd5, 0x88, 0xc0, 0x8f, 0xa1, 0xc1, 0xe8, 0x66, 0x4b, + 0x6c, 0xa4, 0xa9, 0xaa, 0x78, 0xf3, 0xea, 0xf3, 0xcb, 0x6c, 0x2b, 0x6f, 0xf6, 0x57, 0x7e, 0x5a, + 0xb2, 0x01, 0x61, 0xbb, 0x1d, 0x2f, 0xbf, 0xd0, 0x54, 0x6a, 0x48, 0xd2, 0xaf, 0x24, 0xa3, 0x7c, + 0xf0, 0x8f, 0xc7, 0x70, 0xd5, 0xd4, 0xaf, 0xe0, 0xf9, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x59, + 0xb7, 0x03, 0x86, 0x2a, 0x03, 0x00, 0x00, } diff --git a/transformer/request/request.proto b/transformer/request/request.proto index f14f3ea7b..601506796 100644 --- a/transformer/request/request.proto +++ b/transformer/request/request.proto @@ -126,4 +126,10 @@ message Metadata { // prefetch: // https://github.com/WICG/webpackage/blob/master/explainer.md#prefetching-stops-here repeated Preload preloads = 1; + + // Recommended validity duration (`expires - date`), in seconds, of the SXG, + // based on the content being signed. In particular, JS is given a shorter + // lifetime to reduce risk of issues due to downgrades: + // https://wicg.github.io/webpackage/draft-yasskin-http-origin-signed-responses.html#seccons-downgrades. + int32 max_age_secs = 2; } diff --git a/transformer/transformer.go b/transformer/transformer.go index 22d4bb796..258f45912 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -18,8 +18,10 @@ package transformer import ( + "math" "net/url" "regexp" + "strconv" "strings" "github.com/ampproject/amppackager/transformer/internal/amphtml" @@ -48,6 +50,7 @@ var transformerFunctionMap = map[string]func(*transformers.Context) error{ "reorderhead": transformers.ReorderHead, "serversiderendering": transformers.ServerSideRendering, "stripjs": transformers.StripJS, + "stripscriptcomments": transformers.StripScriptComments, "transformedidentifier": transformers.TransformedIdentifier, "unusedextensions": transformers.UnusedExtensions, "urlrewrite": transformers.URLRewrite, @@ -60,6 +63,7 @@ var configMap = map[rpb.Request_TransformersConfig][]func(*transformers.Context) // NodeCleanup should be first. transformers.NodeCleanup, transformers.StripJS, + transformers.StripScriptComments, transformers.LinkTag, transformers.AbsoluteURL, transformers.AMPBoilerplate, @@ -171,6 +175,22 @@ func requireAMPAttribute(dom *amphtml.DOM, allowedFormats []rpb.Request_HtmlForm return errors.New("html tag is missing an AMP attribute") } +// setBaseURL derives the absolute base URL, and sets it on c.BaseURL. The value +// is derived using the href in the DOM, if it exists. If the href is +// relative, it is parsed in the context of the document URL. +// This must run after DocumentURL is set on the context. +func setBaseURL(c *transformers.Context) { + if n, ok := htmlnode.FindNode(c.DOM.HeadNode, atom.Base); ok { + if v, ok := htmlnode.GetAttributeVal(n, "", "href"); ok { + if u, err := c.DocumentURL.Parse(v); err == nil { + c.BaseURL = u + return + } + } + } + c.BaseURL = c.DocumentURL +} + // extractPreloads returns a list of absolute URLs of the resources to preload, // in the order to preload them. It depends on transformers.ReorderHead having // run. @@ -200,20 +220,43 @@ func extractPreloads(dom *amphtml.DOM) []*rpb.Metadata_Preload { return preloads } -// setBaseURL derives the absolute base URL, and sets it on c.BaseURL. The value -// is derived using the href in the DOM, if it exists. If the href is -// relative, it is parsed in the context of the document URL. -// This must run after DocumentURL is set on the context. -func setBaseURL(c *transformers.Context) { - if n, ok := htmlnode.FindNode(c.DOM.HeadNode, atom.Base); ok { - if v, ok := htmlnode.GetAttributeVal(n, "", "href"); ok { - if u, err := c.DocumentURL.Parse(v); err == nil { - c.BaseURL = u - return +// defaultMaxAgeSeconds is the max-age to apply when there is an inline +// amp-script without an explicit max-age. This is 1 day, to parallel the +// security precautions put in place around service workers: +// https://dev.chromium.org/Home/chromium-security/security-faq/service-worker-security-faq#TOC-Do-Service-Workers-live-forever- +const defaultMaxAgeSeconds int32 = 86400 // number of seconds in a day + +// maxMaxAgeSeconds is the max duration of an SXG, per +// https://wicg.github.io/webpackage/draft-yasskin-http-origin-signed-responses.html#signature-validity. +const maxMaxAgeSeconds int32 = 7*86400 + +// computeMaxAgeSeconds returns the suggested max-age based on the presence of +// any inline tags on the page; callers should min() the return +// value against any other they constraints they have (e.g. the max allowed +// duration of an SXG). +func computeMaxAgeSeconds(dom *amphtml.DOM) int32 { + var maxAge int32 = math.MaxInt32 + for node := dom.RootNode; node != nil; node = htmlnode.Next(node) { + // The html parser downcases tag and attribute names, so we needn't. + if node.Type == html.ElementNode && node.Data == "amp-script" && htmlnode.HasAttribute(node, "", "script") { + nodeMaxAge := defaultMaxAgeSeconds + if value, ok := htmlnode.GetAttributeVal(node, "", "max-age"); ok { + if num, err := strconv.ParseInt(value, 10, 32); err == nil { + if num < 0 { + num = 0 + } + nodeMaxAge = int32(num) + } + } + if nodeMaxAge < maxAge { + maxAge = nodeMaxAge } } } - c.BaseURL = c.DocumentURL + if maxAge > maxMaxAgeSeconds { + maxAge = maxMaxAgeSeconds + } + return maxAge } // Process will parse the given request, which contains the HTML to @@ -273,5 +316,9 @@ func Process(r *rpb.Request) (string, *rpb.Metadata, error) { if err := printer.Print(&o, context.DOM.RootNode); err != nil { return "", nil, err } - return o.String(), &rpb.Metadata{Preloads: extractPreloads(context.DOM)}, nil + metadata := rpb.Metadata{ + Preloads: extractPreloads(context.DOM), + MaxAgeSecs: computeMaxAgeSeconds(context.DOM), + } + return o.String(), &metadata, nil } diff --git a/transformer/transformer_test.go b/transformer/transformer_test.go index 046ba9663..32ff55527 100644 --- a/transformer/transformer_test.go +++ b/transformer/transformer_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/golang/protobuf/proto" rpb "github.com/ampproject/amppackager/transformer/request" "github.com/ampproject/amppackager/transformer/transformers" "github.com/google/go-cmp/cmp" @@ -26,7 +27,7 @@ func TestProcess(t *testing.T) { config rpb.Request_TransformersConfig expectedLen int }{ - {rpb.Request_DEFAULT, 12}, + {rpb.Request_DEFAULT, 13}, {rpb.Request_NONE, 0}, {rpb.Request_VALIDATION, 1}, {rpb.Request_CUSTOM, 0}, @@ -109,13 +110,84 @@ func TestPreloads(t *testing.T) { t.Fatalf("unexpected failure: %v", err) } - if diff := cmp.Diff(tc.expectedPreloads, metadata.Preloads); diff != "" { + if diff := cmp.Diff(tc.expectedPreloads, metadata.Preloads, cmp.Comparer(proto.Equal)); diff != "" { t.Errorf("preloads differ (-want +got):\n%s", diff) } }) } } +func TestMaxAge(t *testing.T) { + tcs := []struct { + html string + expectedMaxAgeSecs int32 + }{ + { + // No amp-scripts; no constraints on signing duration. + "", + 604800, + }, + { + // amp-script but not inline; no constraints. + "", + 604800, + }, + { + // Inline amp-script; default to 1-day duration. + "", + 86400, + }, + { + // Inline amp-script with explicit 4-day duration. + "", + 345600, + }, + { + // Inline amp-script with explicit 1-year duration; capped at 7 days. + "", + 604800, + }, + { + // Inline amp-script with invalid duration; use default. + "", + 86400, + }, + { + // Inline amp-script with negative duration; use 0. + "", + 0, + }, + { + // Two inline amp-scripts, use min of both. + "", + 500000, + }, + { + // Two inline amp-scripts, explicit > implicit. + "", + 86400, + }, + { + // Two inline amp-scripts, explicit < implicit. + "", + 1, + }, + } + + for _, tc := range tcs { + t.Run(tc.html, func(t *testing.T) { + _, metadata, err := Process(&rpb.Request{Html: tc.html, Config: rpb.Request_NONE}) + if err != nil { + t.Fatalf("unexpected failure: %v", err) + } + + if metadata.MaxAgeSecs != tc.expectedMaxAgeSecs { + t.Errorf("maxAgeSecs differs; got=%d, want=%d", metadata.MaxAgeSecs, tc.expectedMaxAgeSecs) + } + }) + } +} + func TestVersion(t *testing.T) { // context is the context provided by Process() to runTransformers(). var context *transformers.Context diff --git a/transformer/transformers/ampboilerplate.go b/transformer/transformers/ampboilerplate.go index eaf2e0d56..95c592c43 100644 --- a/transformer/transformers/ampboilerplate.go +++ b/transformer/transformers/ampboilerplate.go @@ -36,6 +36,17 @@ func AMPBoilerplate(e *Context) error { } } + if e.Version >= 3 { + // If the document had been modified by a Server-Side-Rendering transform + // earlier, for example by the AMP Optimizer, and that transform + // determined that the boilerplate was unnecessary, we don't add the + // boilerplate back. Note this can mean that an error in that transform + // could result in boilerplate being removed when it shouldn't be. + if htmlnode.HasAttribute(e.DOM.HTMLNode, "", "i-amphtml-no-boilerplate") { + return nil + } + } + boilerplate, css := determineBoilerplateAndCSS(e.DOM.HTMLNode) styleNode := htmlnode.Element("style", html.Attribute{Key: boilerplate}) diff --git a/transformer/transformers/serversiderendering.go b/transformer/transformers/serversiderendering.go index 7651edc9e..7e5295ad3 100644 --- a/transformer/transformers/serversiderendering.go +++ b/transformer/transformers/serversiderendering.go @@ -133,13 +133,13 @@ func canRemoveBoilerplate(n *html.Node) bool { if n.Data == amphtml.AMPAudio { return false } - if htmlnode.HasAttribute(n, "", "heights") { + if htmlnode.HasAttributeAndIsNotEmpty(n, "", "heights") { return false } - if htmlnode.HasAttribute(n, "", "media") { + if htmlnode.HasAttributeAndIsNotEmpty(n, "", "media") { return false } - if htmlnode.HasAttribute(n, "", "sizes") { + if htmlnode.HasAttributeAndIsNotEmpty(n, "", "sizes") { return false } } diff --git a/transformer/transformers/stripjs.go b/transformer/transformers/stripjs.go index 7d383e5d0..e4e91081d 100644 --- a/transformer/transformers/stripjs.go +++ b/transformer/transformers/stripjs.go @@ -32,9 +32,9 @@ var eventRE = func() *regexp.Regexp { // StripJS removes non-AMP javascript from the DOM. // - For ", Expected: "", }, + { + Desc: "keep type=text/plain", + Input: "", + Expected: "", + }, { Desc: "strip tag attr ona", Input: "