From f8fcffb3e1ca6f2eb0c44dbfc3c858bfe735e048 Mon Sep 17 00:00:00 2001 From: Swapnil Mhamane Date: Sat, 8 Dec 2018 00:02:05 +0530 Subject: [PATCH] Add mock test for GCS Signed-off-by: Swapnil Mhamane --- Gopkg.lock | 64 +- Gopkg.toml | 6 +- LICENSE.md | 15 +- pkg/snapstore/gcs_snapstore.go | 44 +- pkg/snapstore/gcs_snapstore_test.go | 160 ++++ pkg/snapstore/snapstore_test.go | 4 + vendor/cloud.google.com/go/CONTRIBUTORS | 1 + .../go/compute/metadata/metadata.go | 362 ++++--- vendor/cloud.google.com/go/iam/iam.go | 14 +- .../cloud.google.com/go/internal/annotate.go | 2 +- .../go/internal/optional/optional.go | 2 +- vendor/cloud.google.com/go/internal/retry.go | 5 +- .../go/internal/trace/trace.go | 84 ++ .../go/internal/version/version.go | 2 +- vendor/cloud.google.com/go/storage/acl.go | 172 +++- vendor/cloud.google.com/go/storage/bucket.go | 427 +++++++-- vendor/cloud.google.com/go/storage/copy.go | 37 +- vendor/cloud.google.com/go/storage/doc.go | 34 +- vendor/cloud.google.com/go/storage/go110.go | 4 +- vendor/cloud.google.com/go/storage/go17.go | 26 - vendor/cloud.google.com/go/storage/iam.go | 23 +- vendor/cloud.google.com/go/storage/invoke.go | 5 +- .../cloud.google.com/go/storage/not_go110.go | 4 +- .../cloud.google.com/go/storage/not_go17.go | 26 - .../go/storage/notifications.go | 21 +- vendor/cloud.google.com/go/storage/reader.go | 325 ++++++- vendor/cloud.google.com/go/storage/storage.go | 334 +++---- vendor/cloud.google.com/go/storage/writer.go | 56 +- .../aws-sdk-go/aws/awsutil/string_value.go | 25 +- .../aws/credentials/processcreds/provider.go | 425 +++++++++ .../aws/aws-sdk-go/aws/csm/reporter.go | 4 +- .../aws/aws-sdk-go/aws/endpoints/defaults.go | 32 + .../aws/aws-sdk-go/aws/request/handlers.go | 3 + .../aws/aws-sdk-go/aws/request/request.go | 122 ++- .../aws/aws-sdk-go/aws/session/doc.go | 2 +- .../aws/aws-sdk-go/aws/session/session.go | 5 + .../aws-sdk-go/aws/session/shared_config.go | 10 + .../github.com/aws/aws-sdk-go/aws/version.go | 2 +- .../aws/aws-sdk-go/service/s3/api.go | 317 +++++- .../aws-sdk-go/service/s3/customizations.go | 1 + .../google-cloud-go-testing/LICENSE | 202 ++++ .../storage/stiface/adapters.go | 176 ++++ .../storage/stiface/doc.go | 34 + .../storage/stiface/interfaces.go | 133 +++ .../client_golang/prometheus/desc.go | 4 +- .../client_golang/prometheus/registry.go | 12 +- .../client_golang/prometheus/timer.go | 11 +- vendor/golang.org/x/net/http2/transport.go | 25 +- vendor/golang.org/x/oauth2/internal/token.go | 1 + .../x/sys/unix/zsysnum_linux_386.go | 2 +- .../x/sys/unix/zsysnum_linux_amd64.go | 2 +- .../x/sys/unix/zsysnum_linux_arm.go | 2 +- .../x/sys/unix/zsysnum_linux_arm64.go | 2 +- .../x/sys/unix/zsysnum_linux_mips.go | 2 +- .../x/sys/unix/zsysnum_linux_mips64.go | 2 +- .../x/sys/unix/zsysnum_linux_mips64le.go | 2 +- .../x/sys/unix/zsysnum_linux_mipsle.go | 2 +- .../x/sys/unix/zsysnum_linux_ppc64.go | 2 +- .../x/sys/unix/zsysnum_linux_ppc64le.go | 2 +- .../x/sys/unix/zsysnum_linux_riscv64.go | 2 +- .../x/sys/unix/zsysnum_linux_s390x.go | 2 +- .../api/storage/v1/storage-gen.go | 84 +- .../api/annotations/annotations.pb.go | 18 +- .../googleapis/api/annotations/http.pb.go | 35 +- .../googleapis/iam/v1/iam_policy.pb.go | 107 ++- .../genproto/googleapis/iam/v1/policy.pb.go | 50 +- .../genproto/googleapis/rpc/code/code.pb.go | 246 +++++ .../googleapis/rpc/status/status.pb.go | 23 +- vendor/google.golang.org/grpc/balancer.go | 2 +- .../grpc/balancer/balancer.go | 5 +- .../grpc/balancer/base/balancer.go | 8 +- .../grpc/balancer/base/base.go | 12 + .../grpc/balancer/roundrobin/roundrobin.go | 4 +- .../grpc/balancer_conn_wrappers.go | 7 +- .../grpc/balancer_v1_wrapper.go | 2 +- .../grpc_binarylog_v1/binarylog.pb.go | 900 ++++++++++++++++++ vendor/google.golang.org/grpc/call.go | 2 +- vendor/google.golang.org/grpc/clientconn.go | 194 +++- .../grpc/connectivity/connectivity.go | 3 +- .../grpc/credentials/credentials.go | 36 +- .../grpc/credentials/go16.go | 57 -- .../grpc/credentials/go17.go | 59 -- .../grpc/credentials/go18.go | 46 - .../grpc/credentials/internal/syscallconn.go | 61 ++ .../internal/syscallconn_appengine.go} | 18 +- vendor/google.golang.org/grpc/dialoptions.go | 23 +- vendor/google.golang.org/grpc/go16.go | 71 -- vendor/google.golang.org/grpc/go17.go | 72 -- .../google.golang.org/grpc/health/client.go | 107 +++ .../grpc/health/{health.go => server.go} | 24 +- vendor/google.golang.org/grpc/interceptor.go | 2 +- .../grpc/internal/binarylog/binarylog.go | 167 ++++ .../internal/binarylog/binarylog_testutil.go | 42 + .../grpc/internal/binarylog/env_config.go | 210 ++++ .../grpc/internal/binarylog/method_logger.go | 426 +++++++++ .../grpc/internal/binarylog/sink.go | 162 ++++ .../go19.go => internal/binarylog/util.go} | 26 +- .../grpc/internal/channelz/funcs.go | 12 +- .../grpc/internal/channelz/types_linux.go | 2 +- .../grpc/internal/channelz/types_nonlinux.go | 2 +- .../{util_linux_go19.go => util_linux.go} | 2 +- ..._nonlinux_pre_go19.go => util_nonlinux.go} | 2 +- .../grpc/internal/envconfig/envconfig.go | 38 +- .../grpc/internal/grpcsync/event.go | 61 ++ .../grpc/internal/internal.go | 2 +- .../grpc/internal/syscall/syscall_linux.go | 114 +++ .../grpc/internal/syscall/syscall_nonlinux.go | 63 ++ .../grpc/internal/transport/bdp_estimator.go | 7 +- .../grpc/internal/transport/go16.go | 52 - .../grpc/internal/transport/go17.go | 53 -- .../grpc/internal/transport/handler_server.go | 4 +- .../grpc/internal/transport/http2_client.go | 35 +- .../grpc/internal/transport/http2_server.go | 4 +- .../grpc/internal/transport/transport.go | 36 +- .../grpc/metadata/metadata.go | 3 +- .../grpc/naming/dns_resolver.go | 5 +- vendor/google.golang.org/grpc/naming/go17.go | 34 - vendor/google.golang.org/grpc/peer/peer.go | 2 +- .../google.golang.org/grpc/picker_wrapper.go | 2 +- vendor/google.golang.org/grpc/pickfirst.go | 3 +- vendor/google.golang.org/grpc/proxy.go | 52 +- .../grpc/resolver/dns/dns_resolver.go | 34 +- .../grpc/resolver/dns/go19.go | 54 -- .../grpc/resolver/dns/pre_go19.go | 51 - .../grpc/resolver_conn_wrapper.go | 56 +- vendor/google.golang.org/grpc/rpc_util.go | 52 +- vendor/google.golang.org/grpc/server.go | 136 ++- .../google.golang.org/grpc/service_config.go | 17 +- .../google.golang.org/grpc/stats/handlers.go | 3 +- vendor/google.golang.org/grpc/stats/stats.go | 3 +- vendor/google.golang.org/grpc/status/go16.go | 42 - vendor/google.golang.org/grpc/status/go17.go | 44 - .../google.golang.org/grpc/status/status.go | 17 + vendor/google.golang.org/grpc/stream.go | 500 +++++++++- vendor/google.golang.org/grpc/tap/tap.go | 2 +- vendor/google.golang.org/grpc/version.go | 2 +- 136 files changed, 6960 insertions(+), 1783 deletions(-) create mode 100644 pkg/snapstore/gcs_snapstore_test.go create mode 100644 vendor/cloud.google.com/go/internal/trace/trace.go delete mode 100644 vendor/cloud.google.com/go/storage/go17.go delete mode 100644 vendor/cloud.google.com/go/storage/not_go17.go create mode 100644 vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go create mode 100644 vendor/github.com/googleapis/google-cloud-go-testing/LICENSE create mode 100644 vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/adapters.go create mode 100644 vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/doc.go create mode 100644 vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/interfaces.go create mode 100644 vendor/google.golang.org/genproto/googleapis/rpc/code/code.pb.go create mode 100644 vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go delete mode 100644 vendor/google.golang.org/grpc/credentials/go16.go delete mode 100644 vendor/google.golang.org/grpc/credentials/go17.go delete mode 100644 vendor/google.golang.org/grpc/credentials/go18.go create mode 100644 vendor/google.golang.org/grpc/credentials/internal/syscallconn.go rename vendor/google.golang.org/grpc/{naming/go18.go => credentials/internal/syscallconn_appengine.go} (73%) delete mode 100644 vendor/google.golang.org/grpc/go16.go delete mode 100644 vendor/google.golang.org/grpc/go17.go create mode 100644 vendor/google.golang.org/grpc/health/client.go rename vendor/google.golang.org/grpc/health/{health.go => server.go} (81%) create mode 100644 vendor/google.golang.org/grpc/internal/binarylog/binarylog.go create mode 100644 vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go create mode 100644 vendor/google.golang.org/grpc/internal/binarylog/env_config.go create mode 100644 vendor/google.golang.org/grpc/internal/binarylog/method_logger.go create mode 100644 vendor/google.golang.org/grpc/internal/binarylog/sink.go rename vendor/google.golang.org/grpc/{credentials/go19.go => internal/binarylog/util.go} (52%) rename vendor/google.golang.org/grpc/internal/channelz/{util_linux_go19.go => util_linux.go} (96%) rename vendor/google.golang.org/grpc/internal/channelz/{util_nonlinux_pre_go19.go => util_nonlinux.go} (95%) create mode 100644 vendor/google.golang.org/grpc/internal/grpcsync/event.go create mode 100644 vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go create mode 100644 vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go delete mode 100644 vendor/google.golang.org/grpc/internal/transport/go16.go delete mode 100644 vendor/google.golang.org/grpc/internal/transport/go17.go delete mode 100644 vendor/google.golang.org/grpc/naming/go17.go delete mode 100644 vendor/google.golang.org/grpc/resolver/dns/go19.go delete mode 100644 vendor/google.golang.org/grpc/resolver/dns/pre_go19.go delete mode 100644 vendor/google.golang.org/grpc/status/go16.go delete mode 100644 vendor/google.golang.org/grpc/status/go17.go diff --git a/Gopkg.lock b/Gopkg.lock index 8c3d20cfa..c7444081e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,19 +2,20 @@ [[projects]] - digest = "1:16b28cc847407abda1774ae8e56875e91741a70a8da31ceca9d383e648701a66" + digest = "1:c7931499c1ea384e8ddb3d48cb60ecda4136eb84ca8db0b358c5781f5863b9d5" name = "cloud.google.com/go" packages = [ "compute/metadata", "iam", "internal", "internal/optional", + "internal/trace", "internal/version", "storage", ] pruneopts = "NUT" - revision = "20d4028b8a750c2aca76bf9fefa8ed2d0109b573" - version = "v0.19.0" + revision = "74b12019e2aa53ec27882158f59192d7cd6d1998" + version = "v0.33.1" [[projects]] digest = "1:d2ccb697dc13c8fbffafa37baae97594d5592ae8f7e113471084137315536e2b" @@ -33,7 +34,7 @@ version = "0.3.0" [[projects]] - digest = "1:66e9bc21369358ecfe3e1226e1c7040dafb2571b4f857dd9ce2cefb3d21c4dbe" + digest = "1:4a93ee04bd3883778e9c7587618d4a1c175292e0005d85b69d8ec49abf861dfb" name = "github.com/aws/aws-sdk-go" packages = [ "aws", @@ -45,6 +46,7 @@ "aws/credentials", "aws/credentials/ec2rolecreds", "aws/credentials/endpointcreds", + "aws/credentials/processcreds", "aws/credentials/stscreds", "aws/csm", "aws/defaults", @@ -72,8 +74,8 @@ "service/sts", ] pruneopts = "NUT" - revision = "180cc10e5ff368b86dee226b034af7d1672baec6" - version = "v1.15.87" + revision = "130ebb7dc1d6b5a940198b47642fc8b20936a65d" + version = "v1.16.1" [[projects]] branch = "master" @@ -252,6 +254,13 @@ revision = "b001040cd31805261cbd978842099e326dfa857b" version = "v2.0.2" +[[projects]] + digest = "1:5ec3be1c7884398947314546ba477a5c4a56ceab97f0cc551d7fabfe4e4b1067" + name = "github.com/googleapis/google-cloud-go-testing" + packages = ["storage/stiface"] + pruneopts = "NUT" + revision = "f773598aef764db07257d386141bed45e60dc228" + [[projects]] digest = "1:32de9c48a572723e2abbae9ec99544eb3b109eea6ca2e3c7de0093e043116a82" name = "github.com/gophercloud/gophercloud" @@ -427,7 +436,7 @@ revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" [[projects]] - digest = "1:a8512715947d49f24e6aa2d0fdd06f46bdd8d2e792517ba0ac95a2c089eaae28" + digest = "1:7c7cfeecd2b7147bcfec48a4bf622b4879e26aec145a9e373ce51d0c23b16f6b" name = "github.com/prometheus/client_golang" packages = [ "prometheus", @@ -435,8 +444,8 @@ "prometheus/promhttp", ] pruneopts = "NUT" - revision = "abad2d1bd44235a26707c172eab6bca5bf2dbad3" - version = "v0.9.1" + revision = "505eaef017263e299324067d40ca2c48f6a2cf50" + version = "v0.9.2" [[projects]] branch = "master" @@ -469,7 +478,7 @@ "xfs", ] pruneopts = "NUT" - revision = "aa55a523dc0a8297edf51bb75e8eec13eb3be45d" + revision = "1dc9a6cbc91aacc3e8b2d63db4d2e957a5394ac4" [[projects]] digest = "1:9dfe20a82800ffcf0a2bde15beba44cc6d6f6cf1d003420ecc8ad0ac78300d70" @@ -598,11 +607,11 @@ "ssh/terminal", ] pruneopts = "NUT" - revision = "eb0de9b17e854e9b1ccd9963efafc79862359959" + revision = "505ab145d0a99da450461ae2c1a9f6cd10d1f447" [[projects]] branch = "master" - digest = "1:e8e8a00e27478c53779c8b4d1bc4ce91d1b5d59497588a5b644726bece44c1f4" + digest = "1:5ac1e0c3979cd33b29af4d61e7a5fd97d12336d0e8a737a56f7dad5f8d7b7b97" name = "golang.org/x/net" packages = [ "context", @@ -618,11 +627,11 @@ "trace", ] pruneopts = "NUT" - revision = "fae4c4e3ad76c295c3d6d259f898136b4bf833a8" + revision = "610586996380ceef02dd726cc09df7e00a3f8e56" [[projects]] branch = "master" - digest = "1:169c27544e7f54a861a05cd84078b494eaf3e41543f0dfd4dd215fa32139cd40" + digest = "1:5364323a31838682c280a6bae0efe47991de96b9cbfeef4f27b3ca2353eb7ff8" name = "golang.org/x/oauth2" packages = [ ".", @@ -632,18 +641,18 @@ "jwt", ] pruneopts = "NUT" - revision = "28207608b83849a028d4f12e46533a6b6894ecaf" + revision = "d668ce993890a79bda886613ee587a69dd5da7a6" [[projects]] branch = "master" - digest = "1:6ddfd101211f81df3ba1f474baf1c451f7708f01c1e0c4be49cd9f0af03596cf" + digest = "1:d504c2e2a9d57c644347bac08e4e883399a158510e260d21b8ab6b8ea5ef63d5" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "NUT" - revision = "4ed8d59d0b35e1e29334a206d1b3f38b1e5dfb31" + revision = "70b957f3b65e069b4930ea94e2721eefa0f8f695" [[projects]] digest = "1:8a12cbc891b7130d3f660f8a309e5c0b083f831e6ac38cdaa1f12e63c12d6bea" @@ -690,7 +699,7 @@ [[projects]] branch = "master" - digest = "1:0a44042c7a3d3b01cc727cf1654ce0305c6765fa7c7f18c62c22702504613287" + digest = "1:23d9fc277640dc6c36702437a57f0ee033b9f7c9f05a7a425abc7df28f6dd259" name = "google.golang.org/api" packages = [ "gensupport", @@ -705,7 +714,7 @@ "transport/http/internal/propagation", ] pruneopts = "NUT" - revision = "0a71a4356c3f4bcbdd16294c78ca2a31fda36cca" + revision = "1a5ef82f9af45ef51c486291ef2b0a16d82fdb95" [[projects]] digest = "1:b63b351b57e64ae8aec1af030dc5e74a2676fcda62102f006ae4411fac1b04c8" @@ -728,27 +737,30 @@ [[projects]] branch = "master" - digest = "1:19d24f6da9de18caafbf2ecf3b6cc31c75047ef5d9738ef491fb86ef54841090" + digest = "1:a7d48ca460ca1b4f6ccd8c95502443afa05df88aee84de7dbeb667a8754e8fa6" name = "google.golang.org/genproto" packages = [ "googleapis/api/annotations", "googleapis/iam/v1", + "googleapis/rpc/code", "googleapis/rpc/status", ] pruneopts = "NUT" - revision = "31ac5d88444a9e7ad18077db9a165d793ad06a2e" + revision = "bd91e49a0898e27abb88c339b432fa53d7497ac0" [[projects]] - digest = "1:f112c6f3e9bbd16eb113db8f37adb6bd10270142079d00c7ac625534cbb16475" + digest = "1:3f21220da2c9af37c32adb314f1c609a6bf8548d5801bf43b95386af19c148bd" name = "google.golang.org/grpc" packages = [ ".", "balancer", "balancer/base", "balancer/roundrobin", + "binarylog/grpc_binarylog_v1", "codes", "connectivity", "credentials", + "credentials/internal", "encoding", "encoding/proto", "grpclog", @@ -756,9 +768,12 @@ "health/grpc_health_v1", "internal", "internal/backoff", + "internal/binarylog", "internal/channelz", "internal/envconfig", "internal/grpcrand", + "internal/grpcsync", + "internal/syscall", "internal/transport", "keepalive", "metadata", @@ -772,8 +787,8 @@ "tap", ] pruneopts = "NUT" - revision = "2e463a05d100327ca47ac218281906921038fd95" - version = "v1.16.0" + revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8" + version = "v1.17.0" [[projects]] digest = "1:1b91ae0dc69a41d4c2ed23ea5cffb721ea63f5037ca4b81e6d6771fbb8f45129" @@ -840,6 +855,7 @@ "github.com/coreos/etcd/store", "github.com/coreos/etcd/wal", "github.com/coreos/etcd/wal/walpb", + "github.com/googleapis/google-cloud-go-testing/storage/stiface", "github.com/gophercloud/gophercloud", "github.com/gophercloud/gophercloud/openstack", "github.com/gophercloud/gophercloud/openstack/objectstorage/v1/objects", diff --git a/Gopkg.toml b/Gopkg.toml index d77c6919e..391f437d4 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -68,7 +68,11 @@ required = ["github.com/coreos/bbolt"] [[constraint]] name = "cloud.google.com/go" - version = "0.19.0" + version = "0.33.1" + +[[constraint]] + name = "github.com/googleapis/google-cloud-go-testing" + revision = "f773598aef764db07257d386141bed45e60dc228" [[constraint]] name = "github.com/gophercloud/gophercloud" diff --git a/LICENSE.md b/LICENSE.md index 96dcee432..0e3cd0375 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -256,21 +256,26 @@ Copyright (c) 2013-2014 Onsi Fakhouri. MIT License (https://github.com/onsi/gomega/blob/master/LICENSE) Microsoft Azure SDK for Go. -https://github.com/Azure/azure-storage-blob-go -Copyright 2017 Microsoft +https://github.com/Azure/azure-storage-blob-go. +Copyright 2017 Microsoft. MIT License (https://github.com/Azure/azure-storage-blob-go/blob/master/LICENSE) Microsoft Azure SDK for Go. https://github.com/Azure/azure-pipeline-go -Copyright 2017 Microsoft +Copyright 2017 Microsoft. MIT License (https://github.com/Azure/azure-pipeline-go/blob/master/LICENSE) -Google Cloud Client Libraries for Go +Google Cloud Client Libraries for Go. https://github.com/GoogleCloudPlatform/google-cloud-go Copyright 2017 Google Inc. All Rights Reserved. Apache 2 license (https://github.com/GoogleCloudPlatform/google-cloud-go/blob/master/LICENSE) -Gophercloud: an OpenStack SDK for Go +Testing Support for the Google Cloud Client Libraries for Go. +https://github.com/googleapis/google-cloud-go-testing +Copyright 2018 Google Inc. All Rights Reserved. +Apache 2 license (https://github.com/googleapis/google-cloud-go-testing/blob/master/LICENSE) + +Gophercloud: an OpenStack SDK for Go. https://github.com/gophercloud/gophercloud Copyright 2012-2013 Rackspace, Inc. Apache 2 license (https://github.com/gophercloud/gophercloud/blob/master/LICENSE) diff --git a/pkg/snapstore/gcs_snapstore.go b/pkg/snapstore/gcs_snapstore.go index 19d7f1765..18a6263ce 100644 --- a/pkg/snapstore/gcs_snapstore.go +++ b/pkg/snapstore/gcs_snapstore.go @@ -27,53 +27,57 @@ import ( "sync" "cloud.google.com/go/storage" + "github.com/googleapis/google-cloud-go-testing/storage/stiface" "github.com/sirupsen/logrus" "google.golang.org/api/iterator" ) -// GCSSnapStore is snapstore with local disk as backend +// GCSSnapStore is snapstore with local disk as backend. type GCSSnapStore struct { + client stiface.Client prefix string - client *storage.Client bucket string - ctx context.Context // maxParallelChunkUploads hold the maximum number of parallel chunk uploads allowed. maxParallelChunkUploads int tempDir string } const ( - gcsNoOfChunk int64 = 32 //Default configuration in swift installation + gcsNoOfChunk int64 = 32 ) -// NewGCSSnapStore create new S3SnapStore from shared configuration with specified bucket +// NewGCSSnapStore create new GCSSnapStore from shared configuration with specified bucket. func NewGCSSnapStore(bucket, prefix, tempDir string, maxParallelChunkUploads int) (*GCSSnapStore, error) { ctx := context.TODO() - gcsClient, err := storage.NewClient(ctx) + cli, err := storage.NewClient(ctx) if err != nil { return nil, err } + gcsClient := stiface.AdaptClient(cli) + return NewGCSSnapStoreFromClient(bucket, prefix, tempDir, maxParallelChunkUploads, gcsClient), nil +} + +// NewGCSSnapStoreFromClient create new GCSSnapStore from shared configuration with specified bucket. +func NewGCSSnapStoreFromClient(bucket, prefix, tempDir string, maxParallelChunkUploads int, cli stiface.Client) *GCSSnapStore { return &GCSSnapStore{ prefix: prefix, - client: gcsClient, + client: cli, bucket: bucket, - ctx: ctx, maxParallelChunkUploads: maxParallelChunkUploads, tempDir: tempDir, - }, nil - + } } -// Fetch should open reader for the snapshot file from store +// Fetch should open reader for the snapshot file from store. func (s *GCSSnapStore) Fetch(snap Snapshot) (io.ReadCloser, error) { objectName := path.Join(s.prefix, snap.SnapDir, snap.SnapName) - return s.client.Bucket(s.bucket).Object(objectName).NewReader(s.ctx) + ctx := context.TODO() + return s.client.Bucket(s.bucket).Object(objectName).NewReader(ctx) } -// Save will write the snapshot to store +// Save will write the snapshot to store. func (s *GCSSnapStore) Save(snap Snapshot, r io.Reader) error { - // Save it locally tmpfile, err := ioutil.TempFile(s.tempDir, tmpBackupFilePrefix) if err != nil { return fmt.Errorf("failed to create snapshot tempfile: %v", err) @@ -127,7 +131,7 @@ func (s *GCSSnapStore) Save(snap Snapshot, r io.Reader) error { } logrus.Info("All chunk uploaded successfully. Uploading composite object.") bh := s.client.Bucket(s.bucket) - var subObjects []*storage.ObjectHandle + var subObjects []stiface.ObjectHandle for partNumber := int64(1); partNumber <= noOfChunks; partNumber++ { name := path.Join(s.prefix, snap.SnapDir, snap.SnapName, fmt.Sprintf("%010d", partNumber)) obj := bh.Object(name) @@ -190,11 +194,9 @@ func (s *GCSSnapStore) componentUploader(wg *sync.WaitGroup, stopCh <-chan struc } } -// List will list the snapshots from store +// List will list the snapshots from store. func (s *GCSSnapStore) List() (SnapList, error) { - // recursively list all "files", not directory - - it := s.client.Bucket(s.bucket).Objects(s.ctx, &storage.Query{Prefix: s.prefix}) + it := s.client.Bucket(s.bucket).Objects(context.TODO(), &storage.Query{Prefix: s.prefix}) var attrs []*storage.ObjectAttrs for { @@ -225,8 +227,8 @@ func (s *GCSSnapStore) List() (SnapList, error) { return snapList, nil } -// Delete should delete the snapshot file from store +// Delete should delete the snapshot file from store. func (s *GCSSnapStore) Delete(snap Snapshot) error { objectName := path.Join(s.prefix, snap.SnapDir, snap.SnapName) - return s.client.Bucket(s.bucket).Object(objectName).Delete(s.ctx) + return s.client.Bucket(s.bucket).Object(objectName).Delete(context.TODO()) } diff --git a/pkg/snapstore/gcs_snapstore_test.go b/pkg/snapstore/gcs_snapstore_test.go new file mode 100644 index 000000000..f80e8a78d --- /dev/null +++ b/pkg/snapstore/gcs_snapstore_test.go @@ -0,0 +1,160 @@ +// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapstore_test + +import ( + "bytes" + "context" + "fmt" + "io" + "sort" + "sync" + + "cloud.google.com/go/storage" + "github.com/googleapis/google-cloud-go-testing/storage/stiface" + "google.golang.org/api/iterator" +) + +// mockGCSClient is a mock client to be used in unit tests. +type mockGCSClient struct { + stiface.Client + objects map[string]*[]byte + prefix string + objectMutex sync.Mutex +} + +func (m *mockGCSClient) Bucket(name string) stiface.BucketHandle { + return &mockBucketHandle{bucket: name, client: m} +} + +type mockBucketHandle struct { + stiface.BucketHandle + bucket string + client *mockGCSClient +} + +func (m *mockBucketHandle) Object(name string) stiface.ObjectHandle { + return &mockObjectHandle{object: name, client: m.client} +} + +func (m *mockBucketHandle) Objects(context.Context, *storage.Query) stiface.ObjectIterator { + var keys []string + for key, _ := range m.client.objects { + keys = append(keys, key) + } + sort.Strings(keys) + return &mockObjectIterator{keys: keys} +} + +type mockObjectHandle struct { + stiface.ObjectHandle + object string + client *mockGCSClient +} + +func (m *mockObjectHandle) NewReader(ctx context.Context) (stiface.Reader, error) { + if value, ok := m.client.objects[m.object]; ok { + return &mockObjectReader{reader: bytes.NewReader(*value)}, nil + } + return nil, fmt.Errorf("object %s not found", m.object) +} + +func (m *mockObjectHandle) NewWriter(context.Context) stiface.Writer { + return &mockObjectWriter{object: m.object, client: m.client} +} + +func (m *mockObjectHandle) ComposerFrom(objects ...stiface.ObjectHandle) stiface.Composer { + return &mockComposer{ + objectHandles: objects, + client: m.client, + dst: m, + } +} + +func (m *mockObjectHandle) Delete(context.Context) error { + if _, ok := m.client.objects[m.object]; ok { + delete(m.client.objects, m.object) + return nil + } + return fmt.Errorf("object %s not found", m.object) +} + +type mockObjectIterator struct { + stiface.ObjectIterator + currentIndex int + keys []string +} + +func (m *mockObjectIterator) Next() (*storage.ObjectAttrs, error) { + if m.currentIndex < len(m.keys) { + obj := &storage.ObjectAttrs{ + Name: m.keys[m.currentIndex], + } + m.currentIndex++ + return obj, nil + } + return nil, iterator.Done +} + +type mockComposer struct { + stiface.Composer + objectHandles []stiface.ObjectHandle + client *mockGCSClient + dst *mockObjectHandle +} + +func (m *mockComposer) Run(ctx context.Context) (*storage.ObjectAttrs, error) { + dstWriter := m.dst.NewWriter(ctx) + for _, obj := range m.objectHandles { + r, err := obj.NewReader(ctx) + if err != nil { + return nil, err + } + if _, err := io.Copy(dstWriter, r); err != nil { + return nil, err + } + } + return &storage.ObjectAttrs{ + Name: m.dst.object, + }, nil +} + +type mockObjectReader struct { + stiface.Reader + reader io.Reader +} + +func (m *mockObjectReader) Read(p []byte) (n int, err error) { + return m.reader.Read(p) +} + +type mockObjectWriter struct { + stiface.Writer + object string + data []byte + client *mockGCSClient +} + +func (m *mockObjectWriter) Write(p []byte) (n int, err error) { + m.data = append(m.data, p...) + return len(p), nil +} + +func (m *mockObjectWriter) Close() error { + m.client.objectMutex.Lock() + m.client.objects[m.object] = &m.data + m.client.objectMutex.Unlock() + return nil +} diff --git a/pkg/snapstore/snapstore_test.go b/pkg/snapstore/snapstore_test.go index 3c68d9036..53f430352 100644 --- a/pkg/snapstore/snapstore_test.go +++ b/pkg/snapstore/snapstore_test.go @@ -83,6 +83,10 @@ var _ = Describe("Snapstore", func() { }), "swift": NewSwiftSnapstoreFromClient(bucket, prefix, "/tmp", 5, fake.ServiceClient()), "ABS": newFakeABSSnapstore(), + "GCS": NewGCSSnapStoreFromClient(bucket, prefix, "/tmp", 5, &mockGCSClient{ + objects: objectMap, + prefix: prefix, + }), } }) diff --git a/vendor/cloud.google.com/go/CONTRIBUTORS b/vendor/cloud.google.com/go/CONTRIBUTORS index eb55275bf..3b3cbed98 100644 --- a/vendor/cloud.google.com/go/CONTRIBUTORS +++ b/vendor/cloud.google.com/go/CONTRIBUTORS @@ -22,6 +22,7 @@ David Symonds Filippo Valsorda Glenn Lewis Ingo Oeser +James Hall Johan Euphrosine Jonathan Amsterdam Kunpei Sakai diff --git a/vendor/cloud.google.com/go/compute/metadata/metadata.go b/vendor/cloud.google.com/go/compute/metadata/metadata.go index e708c031b..0d929a619 100644 --- a/vendor/cloud.google.com/go/compute/metadata/metadata.go +++ b/vendor/cloud.google.com/go/compute/metadata/metadata.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ package metadata // import "cloud.google.com/go/compute/metadata" import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -31,9 +32,6 @@ import ( "strings" "sync" "time" - - "golang.org/x/net/context" - "golang.org/x/net/context/ctxhttp" ) const ( @@ -64,7 +62,7 @@ var ( ) var ( - metaClient = &http.Client{ + defaultClient = &Client{hc: &http.Client{ Transport: &http.Transport{ Dial: (&net.Dialer{ Timeout: 2 * time.Second, @@ -72,15 +70,15 @@ var ( }).Dial, ResponseHeaderTimeout: 2 * time.Second, }, - } - subscribeClient = &http.Client{ + }} + subscribeClient = &Client{hc: &http.Client{ Transport: &http.Transport{ Dial: (&net.Dialer{ Timeout: 2 * time.Second, KeepAlive: 30 * time.Second, }).Dial, }, - } + }} ) // NotDefinedError is returned when requested metadata is not defined. @@ -95,74 +93,16 @@ func (suffix NotDefinedError) Error() string { return fmt.Sprintf("metadata: GCE metadata %q not defined", string(suffix)) } -// Get returns a value from the metadata service. -// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". -// -// If the GCE_METADATA_HOST environment variable is not defined, a default of -// 169.254.169.254 will be used instead. -// -// If the requested metadata is not defined, the returned error will -// be of type NotDefinedError. -func Get(suffix string) (string, error) { - val, _, err := getETag(metaClient, suffix) - return val, err -} - -// getETag returns a value from the metadata service as well as the associated -// ETag using the provided client. This func is otherwise equivalent to Get. -func getETag(client *http.Client, suffix string) (value, etag string, err error) { - // Using a fixed IP makes it very difficult to spoof the metadata service in - // a container, which is an important use-case for local testing of cloud - // deployments. To enable spoofing of the metadata service, the environment - // variable GCE_METADATA_HOST is first inspected to decide where metadata - // requests shall go. - host := os.Getenv(metadataHostEnv) - if host == "" { - // Using 169.254.169.254 instead of "metadata" here because Go - // binaries built with the "netgo" tag and without cgo won't - // know the search suffix for "metadata" is - // ".google.internal", and this IP address is documented as - // being stable anyway. - host = metadataIP - } - url := "http://" + host + "/computeMetadata/v1/" + suffix - req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Metadata-Flavor", "Google") - req.Header.Set("User-Agent", userAgent) - res, err := client.Do(req) - if err != nil { - return "", "", err - } - defer res.Body.Close() - if res.StatusCode == http.StatusNotFound { - return "", "", NotDefinedError(suffix) - } - if res.StatusCode != 200 { - return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) - } - all, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", "", err - } - return string(all), res.Header.Get("Etag"), nil -} - -func getTrimmed(suffix string) (s string, err error) { - s, err = Get(suffix) - s = strings.TrimSpace(s) - return -} - -func (c *cachedValue) get() (v string, err error) { +func (c *cachedValue) get(cl *Client) (v string, err error) { defer c.mu.Unlock() c.mu.Lock() if c.v != "" { return c.v, nil } if c.trim { - v, err = getTrimmed(c.k) + v, err = cl.getTrimmed(c.k) } else { - v, err = Get(c.k) + v, err = cl.Get(c.k) } if err == nil { c.v = v @@ -201,7 +141,7 @@ func testOnGCE() bool { go func() { req, _ := http.NewRequest("GET", "http://"+metadataIP, nil) req.Header.Set("User-Agent", userAgent) - res, err := ctxhttp.Do(ctx, metaClient, req) + res, err := defaultClient.hc.Do(req.WithContext(ctx)) if err != nil { resc <- false return @@ -266,78 +206,183 @@ func systemInfoSuggestsGCE() bool { return name == "Google" || name == "Google Compute Engine" } -// Subscribe subscribes to a value from the metadata service. -// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". -// The suffix may contain query parameters. -// -// Subscribe calls fn with the latest metadata value indicated by the provided -// suffix. If the metadata value is deleted, fn is called with the empty string -// and ok false. Subscribe blocks until fn returns a non-nil error or the value -// is deleted. Subscribe returns the error value returned from the last call to -// fn, which may be nil when ok == false. +// Subscribe calls Client.Subscribe on a client designed for subscribing (one with no +// ResponseHeaderTimeout). func Subscribe(suffix string, fn func(v string, ok bool) error) error { - const failedSubscribeSleep = time.Second * 5 + return subscribeClient.Subscribe(suffix, fn) +} - // First check to see if the metadata value exists at all. - val, lastETag, err := getETag(subscribeClient, suffix) - if err != nil { - return err - } +// Get calls Client.Get on the default client. +func Get(suffix string) (string, error) { return defaultClient.Get(suffix) } - if err := fn(val, true); err != nil { - return err +// ProjectID returns the current instance's project ID string. +func ProjectID() (string, error) { return defaultClient.ProjectID() } + +// NumericProjectID returns the current instance's numeric project ID. +func NumericProjectID() (string, error) { return defaultClient.NumericProjectID() } + +// InternalIP returns the instance's primary internal IP address. +func InternalIP() (string, error) { return defaultClient.InternalIP() } + +// ExternalIP returns the instance's primary external (public) IP address. +func ExternalIP() (string, error) { return defaultClient.ExternalIP() } + +// Hostname returns the instance's hostname. This will be of the form +// ".c..internal". +func Hostname() (string, error) { return defaultClient.Hostname() } + +// InstanceTags returns the list of user-defined instance tags, +// assigned when initially creating a GCE instance. +func InstanceTags() ([]string, error) { return defaultClient.InstanceTags() } + +// InstanceID returns the current VM's numeric instance ID. +func InstanceID() (string, error) { return defaultClient.InstanceID() } + +// InstanceName returns the current VM's instance ID string. +func InstanceName() (string, error) { return defaultClient.InstanceName() } + +// Zone returns the current VM's zone, such as "us-central1-b". +func Zone() (string, error) { return defaultClient.Zone() } + +// InstanceAttributes calls Client.InstanceAttributes on the default client. +func InstanceAttributes() ([]string, error) { return defaultClient.InstanceAttributes() } + +// ProjectAttributes calls Client.ProjectAttributes on the default client. +func ProjectAttributes() ([]string, error) { return defaultClient.ProjectAttributes() } + +// InstanceAttributeValue calls Client.InstanceAttributeValue on the default client. +func InstanceAttributeValue(attr string) (string, error) { + return defaultClient.InstanceAttributeValue(attr) +} + +// ProjectAttributeValue calls Client.ProjectAttributeValue on the default client. +func ProjectAttributeValue(attr string) (string, error) { + return defaultClient.ProjectAttributeValue(attr) +} + +// Scopes calls Client.Scopes on the default client. +func Scopes(serviceAccount string) ([]string, error) { return defaultClient.Scopes(serviceAccount) } + +func strsContains(ss []string, s string) bool { + for _, v := range ss { + if v == s { + return true + } } + return false +} - ok := true - if strings.ContainsRune(suffix, '?') { - suffix += "&wait_for_change=true&last_etag=" - } else { - suffix += "?wait_for_change=true&last_etag=" +// A Client provides metadata. +type Client struct { + hc *http.Client +} + +// NewClient returns a Client that can be used to fetch metadata. All HTTP requests +// will use the given http.Client instead of the default client. +func NewClient(c *http.Client) *Client { + return &Client{hc: c} +} + +// getETag returns a value from the metadata service as well as the associated ETag. +// This func is otherwise equivalent to Get. +func (c *Client) getETag(suffix string) (value, etag string, err error) { + // Using a fixed IP makes it very difficult to spoof the metadata service in + // a container, which is an important use-case for local testing of cloud + // deployments. To enable spoofing of the metadata service, the environment + // variable GCE_METADATA_HOST is first inspected to decide where metadata + // requests shall go. + host := os.Getenv(metadataHostEnv) + if host == "" { + // Using 169.254.169.254 instead of "metadata" here because Go + // binaries built with the "netgo" tag and without cgo won't + // know the search suffix for "metadata" is + // ".google.internal", and this IP address is documented as + // being stable anyway. + host = metadataIP } - for { - val, etag, err := getETag(subscribeClient, suffix+url.QueryEscape(lastETag)) - if err != nil { - if _, deleted := err.(NotDefinedError); !deleted { - time.Sleep(failedSubscribeSleep) - continue // Retry on other errors. - } - ok = false - } - lastETag = etag + url := "http://" + host + "/computeMetadata/v1/" + suffix + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Metadata-Flavor", "Google") + req.Header.Set("User-Agent", userAgent) + res, err := c.hc.Do(req) + if err != nil { + return "", "", err + } + defer res.Body.Close() + if res.StatusCode == http.StatusNotFound { + return "", "", NotDefinedError(suffix) + } + if res.StatusCode != 200 { + return "", "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + return "", "", err + } + return string(all), res.Header.Get("Etag"), nil +} - if err := fn(val, ok); err != nil || !ok { - return err - } +// Get returns a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// +// If the GCE_METADATA_HOST environment variable is not defined, a default of +// 169.254.169.254 will be used instead. +// +// If the requested metadata is not defined, the returned error will +// be of type NotDefinedError. +func (c *Client) Get(suffix string) (string, error) { + val, _, err := c.getETag(suffix) + return val, err +} + +func (c *Client) getTrimmed(suffix string) (s string, err error) { + s, err = c.Get(suffix) + s = strings.TrimSpace(s) + return +} + +func (c *Client) lines(suffix string) ([]string, error) { + j, err := c.Get(suffix) + if err != nil { + return nil, err } + s := strings.Split(strings.TrimSpace(j), "\n") + for i := range s { + s[i] = strings.TrimSpace(s[i]) + } + return s, nil } // ProjectID returns the current instance's project ID string. -func ProjectID() (string, error) { return projID.get() } +func (c *Client) ProjectID() (string, error) { return projID.get(c) } // NumericProjectID returns the current instance's numeric project ID. -func NumericProjectID() (string, error) { return projNum.get() } +func (c *Client) NumericProjectID() (string, error) { return projNum.get(c) } + +// InstanceID returns the current VM's numeric instance ID. +func (c *Client) InstanceID() (string, error) { return instID.get(c) } // InternalIP returns the instance's primary internal IP address. -func InternalIP() (string, error) { - return getTrimmed("instance/network-interfaces/0/ip") +func (c *Client) InternalIP() (string, error) { + return c.getTrimmed("instance/network-interfaces/0/ip") } // ExternalIP returns the instance's primary external (public) IP address. -func ExternalIP() (string, error) { - return getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") +func (c *Client) ExternalIP() (string, error) { + return c.getTrimmed("instance/network-interfaces/0/access-configs/0/external-ip") } // Hostname returns the instance's hostname. This will be of the form // ".c..internal". -func Hostname() (string, error) { - return getTrimmed("instance/hostname") +func (c *Client) Hostname() (string, error) { + return c.getTrimmed("instance/hostname") } // InstanceTags returns the list of user-defined instance tags, // assigned when initially creating a GCE instance. -func InstanceTags() ([]string, error) { +func (c *Client) InstanceTags() ([]string, error) { var s []string - j, err := Get("instance/tags") + j, err := c.Get("instance/tags") if err != nil { return nil, err } @@ -347,14 +392,9 @@ func InstanceTags() ([]string, error) { return s, nil } -// InstanceID returns the current VM's numeric instance ID. -func InstanceID() (string, error) { - return instID.get() -} - // InstanceName returns the current VM's instance ID string. -func InstanceName() (string, error) { - host, err := Hostname() +func (c *Client) InstanceName() (string, error) { + host, err := c.Hostname() if err != nil { return "", err } @@ -362,8 +402,8 @@ func InstanceName() (string, error) { } // Zone returns the current VM's zone, such as "us-central1-b". -func Zone() (string, error) { - zone, err := getTrimmed("instance/zone") +func (c *Client) Zone() (string, error) { + zone, err := c.getTrimmed("instance/zone") // zone is of the form "projects//zones/". if err != nil { return "", err @@ -374,24 +414,12 @@ func Zone() (string, error) { // InstanceAttributes returns the list of user-defined attributes, // assigned when initially creating a GCE VM instance. The value of an // attribute can be obtained with InstanceAttributeValue. -func InstanceAttributes() ([]string, error) { return lines("instance/attributes/") } +func (c *Client) InstanceAttributes() ([]string, error) { return c.lines("instance/attributes/") } // ProjectAttributes returns the list of user-defined attributes // applying to the project as a whole, not just this VM. The value of // an attribute can be obtained with ProjectAttributeValue. -func ProjectAttributes() ([]string, error) { return lines("project/attributes/") } - -func lines(suffix string) ([]string, error) { - j, err := Get(suffix) - if err != nil { - return nil, err - } - s := strings.Split(strings.TrimSpace(j), "\n") - for i := range s { - s[i] = strings.TrimSpace(s[i]) - } - return s, nil -} +func (c *Client) ProjectAttributes() ([]string, error) { return c.lines("project/attributes/") } // InstanceAttributeValue returns the value of the provided VM // instance attribute. @@ -401,8 +429,8 @@ func lines(suffix string) ([]string, error) { // // InstanceAttributeValue may return ("", nil) if the attribute was // defined to be the empty string. -func InstanceAttributeValue(attr string) (string, error) { - return Get("instance/attributes/" + attr) +func (c *Client) InstanceAttributeValue(attr string) (string, error) { + return c.Get("instance/attributes/" + attr) } // ProjectAttributeValue returns the value of the provided @@ -413,25 +441,61 @@ func InstanceAttributeValue(attr string) (string, error) { // // ProjectAttributeValue may return ("", nil) if the attribute was // defined to be the empty string. -func ProjectAttributeValue(attr string) (string, error) { - return Get("project/attributes/" + attr) +func (c *Client) ProjectAttributeValue(attr string) (string, error) { + return c.Get("project/attributes/" + attr) } // Scopes returns the service account scopes for the given account. // The account may be empty or the string "default" to use the instance's // main account. -func Scopes(serviceAccount string) ([]string, error) { +func (c *Client) Scopes(serviceAccount string) ([]string, error) { if serviceAccount == "" { serviceAccount = "default" } - return lines("instance/service-accounts/" + serviceAccount + "/scopes") + return c.lines("instance/service-accounts/" + serviceAccount + "/scopes") } -func strsContains(ss []string, s string) bool { - for _, v := range ss { - if v == s { - return true +// Subscribe subscribes to a value from the metadata service. +// The suffix is appended to "http://${GCE_METADATA_HOST}/computeMetadata/v1/". +// The suffix may contain query parameters. +// +// Subscribe calls fn with the latest metadata value indicated by the provided +// suffix. If the metadata value is deleted, fn is called with the empty string +// and ok false. Subscribe blocks until fn returns a non-nil error or the value +// is deleted. Subscribe returns the error value returned from the last call to +// fn, which may be nil when ok == false. +func (c *Client) Subscribe(suffix string, fn func(v string, ok bool) error) error { + const failedSubscribeSleep = time.Second * 5 + + // First check to see if the metadata value exists at all. + val, lastETag, err := c.getETag(suffix) + if err != nil { + return err + } + + if err := fn(val, true); err != nil { + return err + } + + ok := true + if strings.ContainsRune(suffix, '?') { + suffix += "&wait_for_change=true&last_etag=" + } else { + suffix += "?wait_for_change=true&last_etag=" + } + for { + val, etag, err := c.getETag(suffix + url.QueryEscape(lastETag)) + if err != nil { + if _, deleted := err.(NotDefinedError); !deleted { + time.Sleep(failedSubscribeSleep) + continue // Retry on other errors. + } + ok = false + } + lastETag = etag + + if err := fn(val, ok); err != nil || !ok { + return err } } - return false } diff --git a/vendor/cloud.google.com/go/iam/iam.go b/vendor/cloud.google.com/go/iam/iam.go index 37720aa2d..76df63561 100644 --- a/vendor/cloud.google.com/go/iam/iam.go +++ b/vendor/cloud.google.com/go/iam/iam.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,10 +22,10 @@ package iam import ( + "context" "time" gax "github.com/googleapis/gax-go" - "golang.org/x/net/context" pb "google.golang.org/genproto/googleapis/iam/v1" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -104,7 +104,15 @@ type Handle struct { // InternalNewHandle returns a Handle for resource. // The conn parameter refers to a server that must support the IAMPolicy service. func InternalNewHandle(conn *grpc.ClientConn, resource string) *Handle { - return InternalNewHandleClient(&grpcClient{c: pb.NewIAMPolicyClient(conn)}, resource) + return InternalNewHandleGRPCClient(pb.NewIAMPolicyClient(conn), resource) +} + +// InternalNewHandleGRPCClient is for use by the Google Cloud Libraries only. +// +// InternalNewHandleClient returns a Handle for resource using the given +// grpc service that implements IAM as a mixin +func InternalNewHandleGRPCClient(c pb.IAMPolicyClient, resource string) *Handle { + return InternalNewHandleClient(&grpcClient{c: c}, resource) } // InternalNewHandleClient is for use by the Google Cloud Libraries only. diff --git a/vendor/cloud.google.com/go/internal/annotate.go b/vendor/cloud.google.com/go/internal/annotate.go index 797809aed..6435695ba 100644 --- a/vendor/cloud.google.com/go/internal/annotate.go +++ b/vendor/cloud.google.com/go/internal/annotate.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/internal/optional/optional.go b/vendor/cloud.google.com/go/internal/optional/optional.go index 4c15410aa..72780f764 100644 --- a/vendor/cloud.google.com/go/internal/optional/optional.go +++ b/vendor/cloud.google.com/go/internal/optional/optional.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/internal/retry.go b/vendor/cloud.google.com/go/internal/retry.go index e1f9aaad6..99093cc46 100644 --- a/vendor/cloud.google.com/go/internal/retry.go +++ b/vendor/cloud.google.com/go/internal/retry.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,10 @@ package internal import ( + "context" "time" gax "github.com/googleapis/gax-go" - - "golang.org/x/net/context" ) // Retry calls the supplied function f repeatedly according to the provided diff --git a/vendor/cloud.google.com/go/internal/trace/trace.go b/vendor/cloud.google.com/go/internal/trace/trace.go new file mode 100644 index 000000000..95c7821c2 --- /dev/null +++ b/vendor/cloud.google.com/go/internal/trace/trace.go @@ -0,0 +1,84 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package trace + +import ( + "context" + + "go.opencensus.io/trace" + "google.golang.org/api/googleapi" + "google.golang.org/genproto/googleapis/rpc/code" + "google.golang.org/grpc/status" +) + +// StartSpan adds a span to the trace with the given name. +func StartSpan(ctx context.Context, name string) context.Context { + ctx, _ = trace.StartSpan(ctx, name) + return ctx +} + +// EndSpan ends a span with the given error. +func EndSpan(ctx context.Context, err error) { + span := trace.FromContext(ctx) + if err != nil { + span.SetStatus(toStatus(err)) + } + span.End() +} + +// ToStatus interrogates an error and converts it to an appropriate +// OpenCensus status. +func toStatus(err error) trace.Status { + if err2, ok := err.(*googleapi.Error); ok { + return trace.Status{Code: httpStatusCodeToOCCode(err2.Code), Message: err2.Message} + } else if s, ok := status.FromError(err); ok { + return trace.Status{Code: int32(s.Code()), Message: s.Message()} + } else { + return trace.Status{Code: int32(code.Code_UNKNOWN), Message: err.Error()} + } +} + +// TODO (deklerk): switch to using OpenCensus function when it becomes available. +// Reference: https://github.com/googleapis/googleapis/blob/26b634d2724ac5dd30ae0b0cbfb01f07f2e4050e/google/rpc/code.proto +func httpStatusCodeToOCCode(httpStatusCode int) int32 { + switch httpStatusCode { + case 200: + return int32(code.Code_OK) + case 499: + return int32(code.Code_CANCELLED) + case 500: + return int32(code.Code_UNKNOWN) // Could also be Code_INTERNAL, Code_DATA_LOSS + case 400: + return int32(code.Code_INVALID_ARGUMENT) // Could also be Code_OUT_OF_RANGE + case 504: + return int32(code.Code_DEADLINE_EXCEEDED) + case 404: + return int32(code.Code_NOT_FOUND) + case 409: + return int32(code.Code_ALREADY_EXISTS) // Could also be Code_ABORTED + case 403: + return int32(code.Code_PERMISSION_DENIED) + case 401: + return int32(code.Code_UNAUTHENTICATED) + case 429: + return int32(code.Code_RESOURCE_EXHAUSTED) + case 501: + return int32(code.Code_UNIMPLEMENTED) + case 503: + return int32(code.Code_UNAVAILABLE) + default: + return int32(code.Code_UNKNOWN) + } +} diff --git a/vendor/cloud.google.com/go/internal/version/version.go b/vendor/cloud.google.com/go/internal/version/version.go index f5c23a564..220f02c1e 100644 --- a/vendor/cloud.google.com/go/internal/version/version.go +++ b/vendor/cloud.google.com/go/internal/version/version.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/vendor/cloud.google.com/go/storage/acl.go b/vendor/cloud.google.com/go/storage/acl.go index 24f90c924..7855d110a 100644 --- a/vendor/cloud.google.com/go/storage/acl.go +++ b/vendor/cloud.google.com/go/storage/acl.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,10 +15,11 @@ package storage import ( + "context" "net/http" "reflect" - "golang.org/x/net/context" + "cloud.google.com/go/internal/trace" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" ) @@ -47,10 +48,21 @@ const ( AllAuthenticatedUsers ACLEntity = "allAuthenticatedUsers" ) -// ACLRule represents a grant for a role to an entity (user, group or team) for a Google Cloud Storage object or bucket. +// ACLRule represents a grant for a role to an entity (user, group or team) for a +// Google Cloud Storage object or bucket. type ACLRule struct { - Entity ACLEntity - Role ACLRole + Entity ACLEntity + EntityID string + Role ACLRole + Domain string + Email string + ProjectTeam *ProjectTeam +} + +// ProjectTeam is the project team associated with the entity, if any. +type ProjectTeam struct { + ProjectNumber string + Team string } // ACLHandle provides operations on an access control list for a Google Cloud Storage bucket or object. @@ -63,7 +75,10 @@ type ACLHandle struct { } // Delete permanently deletes the ACL entry for the given entity. -func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error { +func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.Delete") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectDelete(ctx, entity) } @@ -73,8 +88,11 @@ func (a *ACLHandle) Delete(ctx context.Context, entity ACLEntity) error { return a.bucketDelete(ctx, entity) } -// Set sets the permission level for the given entity. -func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) error { +// Set sets the role for the given entity. +func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.Set") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectSet(ctx, entity, role, false) } @@ -85,7 +103,10 @@ func (a *ACLHandle) Set(ctx context.Context, entity ACLEntity, role ACLRole) err } // List retrieves ACL entries. -func (a *ACLHandle) List(ctx context.Context) ([]ACLRule, error) { +func (a *ACLHandle) List(ctx context.Context) (rules []ACLRule, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.ACL.List") + defer func() { trace.EndSpan(ctx, err) }() + if a.object != "" { return a.objectList(ctx) } @@ -100,20 +121,20 @@ func (a *ACLHandle) bucketDefaultList(ctx context.Context) ([]ACLRule, error) { var err error err = runWithRetry(ctx, func() error { req := a.c.raw.DefaultObjectAccessControls.List(a.bucket) - a.configureCall(req, ctx) + a.configureCall(ctx, req) acls, err = req.Do() return err }) if err != nil { return nil, err } - return toACLRules(acls.Items), nil + return toObjectACLRules(acls.Items), nil } func (a *ACLHandle) bucketDefaultDelete(ctx context.Context, entity ACLEntity) error { return runWithRetry(ctx, func() error { req := a.c.raw.DefaultObjectAccessControls.Delete(a.bucket, string(entity)) - a.configureCall(req, ctx) + a.configureCall(ctx, req) return req.Do() }) } @@ -123,19 +144,14 @@ func (a *ACLHandle) bucketList(ctx context.Context) ([]ACLRule, error) { var err error err = runWithRetry(ctx, func() error { req := a.c.raw.BucketAccessControls.List(a.bucket) - a.configureCall(req, ctx) + a.configureCall(ctx, req) acls, err = req.Do() return err }) if err != nil { return nil, err } - r := make([]ACLRule, len(acls.Items)) - for i, v := range acls.Items { - r[i].Entity = ACLEntity(v.Entity) - r[i].Role = ACLRole(v.Role) - } - return r, nil + return toBucketACLRules(acls.Items), nil } func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRole) error { @@ -146,7 +162,7 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol } err := runWithRetry(ctx, func() error { req := a.c.raw.BucketAccessControls.Update(a.bucket, string(entity), acl) - a.configureCall(req, ctx) + a.configureCall(ctx, req) _, err := req.Do() return err }) @@ -157,15 +173,11 @@ func (a *ACLHandle) bucketSet(ctx context.Context, entity ACLEntity, role ACLRol } func (a *ACLHandle) bucketDelete(ctx context.Context, entity ACLEntity) error { - err := runWithRetry(ctx, func() error { + return runWithRetry(ctx, func() error { req := a.c.raw.BucketAccessControls.Delete(a.bucket, string(entity)) - a.configureCall(req, ctx) + a.configureCall(ctx, req) return req.Do() }) - if err != nil { - return err - } - return nil } func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { @@ -173,14 +185,14 @@ func (a *ACLHandle) objectList(ctx context.Context) ([]ACLRule, error) { var err error err = runWithRetry(ctx, func() error { req := a.c.raw.ObjectAccessControls.List(a.bucket, a.object) - a.configureCall(req, ctx) + a.configureCall(ctx, req) acls, err = req.Do() return err }) if err != nil { return nil, err } - return toACLRules(acls.Items), nil + return toObjectACLRules(acls.Items), nil } func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRole, isBucketDefault bool) error { @@ -200,7 +212,7 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol } else { req = a.c.raw.ObjectAccessControls.Update(a.bucket, a.object, string(entity), acl) } - a.configureCall(req, ctx) + a.configureCall(ctx, req) return runWithRetry(ctx, func() error { _, err := req.Do() return err @@ -210,14 +222,12 @@ func (a *ACLHandle) objectSet(ctx context.Context, entity ACLEntity, role ACLRol func (a *ACLHandle) objectDelete(ctx context.Context, entity ACLEntity) error { return runWithRetry(ctx, func() error { req := a.c.raw.ObjectAccessControls.Delete(a.bucket, a.object, string(entity)) - a.configureCall(req, ctx) + a.configureCall(ctx, req) return req.Do() }) } -func (a *ACLHandle) configureCall(call interface { - Header() http.Header -}, ctx context.Context) { +func (a *ACLHandle) configureCall(ctx context.Context, call interface{ Header() http.Header }) { vc := reflect.ValueOf(call) vc.MethodByName("Context").Call([]reflect.Value{reflect.ValueOf(ctx)}) if a.userProject != "" { @@ -226,10 +236,100 @@ func (a *ACLHandle) configureCall(call interface { setClientHeader(call.Header()) } -func toACLRules(items []*raw.ObjectAccessControl) []ACLRule { - r := make([]ACLRule, 0, len(items)) +func toObjectACLRules(items []*raw.ObjectAccessControl) []ACLRule { + var rs []ACLRule for _, item := range items { - r = append(r, ACLRule{Entity: ACLEntity(item.Entity), Role: ACLRole(item.Role)}) + rs = append(rs, toObjectACLRule(item)) + } + return rs +} + +func toBucketACLRules(items []*raw.BucketAccessControl) []ACLRule { + var rs []ACLRule + for _, item := range items { + rs = append(rs, toBucketACLRule(item)) + } + return rs +} + +func toObjectACLRule(a *raw.ObjectAccessControl) ACLRule { + return ACLRule{ + Entity: ACLEntity(a.Entity), + EntityID: a.EntityId, + Role: ACLRole(a.Role), + Domain: a.Domain, + Email: a.Email, + ProjectTeam: toObjectProjectTeam(a.ProjectTeam), + } +} + +func toBucketACLRule(a *raw.BucketAccessControl) ACLRule { + return ACLRule{ + Entity: ACLEntity(a.Entity), + EntityID: a.EntityId, + Role: ACLRole(a.Role), + Domain: a.Domain, + Email: a.Email, + ProjectTeam: toBucketProjectTeam(a.ProjectTeam), + } +} + +func toRawObjectACL(rules []ACLRule) []*raw.ObjectAccessControl { + if len(rules) == 0 { + return nil + } + r := make([]*raw.ObjectAccessControl, 0, len(rules)) + for _, rule := range rules { + r = append(r, rule.toRawObjectAccessControl("")) // bucket name unnecessary } return r } + +func toRawBucketACL(rules []ACLRule) []*raw.BucketAccessControl { + if len(rules) == 0 { + return nil + } + r := make([]*raw.BucketAccessControl, 0, len(rules)) + for _, rule := range rules { + r = append(r, rule.toRawBucketAccessControl("")) // bucket name unnecessary + } + return r +} + +func (r ACLRule) toRawBucketAccessControl(bucket string) *raw.BucketAccessControl { + return &raw.BucketAccessControl{ + Bucket: bucket, + Entity: string(r.Entity), + Role: string(r.Role), + // The other fields are not settable. + } +} + +func (r ACLRule) toRawObjectAccessControl(bucket string) *raw.ObjectAccessControl { + return &raw.ObjectAccessControl{ + Bucket: bucket, + Entity: string(r.Entity), + Role: string(r.Role), + // The other fields are not settable. + } +} + +func toBucketProjectTeam(p *raw.BucketAccessControlProjectTeam) *ProjectTeam { + if p == nil { + return nil + } + return &ProjectTeam{ + ProjectNumber: p.ProjectNumber, + Team: p.Team, + } +} + +func toObjectProjectTeam(p *raw.ObjectAccessControlProjectTeam) *ProjectTeam { + if p == nil { + return nil + } + return &ProjectTeam{ + ProjectNumber: p.ProjectNumber, + Team: p.Team, + } +} diff --git a/vendor/cloud.google.com/go/storage/bucket.go b/vendor/cloud.google.com/go/storage/bucket.go index fa802b58b..3b0018aff 100644 --- a/vendor/cloud.google.com/go/storage/bucket.go +++ b/vendor/cloud.google.com/go/storage/bucket.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. LiveAndArchived Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,13 +15,14 @@ package storage import ( + "context" "fmt" "net/http" "reflect" "time" "cloud.google.com/go/internal/optional" - "golang.org/x/net/context" + "cloud.google.com/go/internal/trace" "google.golang.org/api/googleapi" "google.golang.org/api/iterator" raw "google.golang.org/api/storage/v1" @@ -63,7 +64,10 @@ func (c *Client) Bucket(name string) *BucketHandle { // Create creates the Bucket in the project. // If attrs is nil the API defaults will be used. -func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) error { +func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *BucketAttrs) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") + defer func() { trace.EndSpan(ctx, err) }() + var bkt *raw.Bucket if attrs != nil { bkt = attrs.toRawBucket() @@ -78,11 +82,20 @@ func (b *BucketHandle) Create(ctx context.Context, projectID string, attrs *Buck } req := b.c.raw.Buckets.Insert(projectID, bkt) setClientHeader(req.Header()) + if attrs != nil && attrs.PredefinedACL != "" { + req.PredefinedAcl(attrs.PredefinedACL) + } + if attrs != nil && attrs.PredefinedDefaultObjectACL != "" { + req.PredefinedDefaultObjectAcl(attrs.PredefinedDefaultObjectACL) + } return runWithRetry(ctx, func() error { _, err := req.Context(ctx).Do(); return err }) } // Delete deletes the Bucket. -func (b *BucketHandle) Delete(ctx context.Context) error { +func (b *BucketHandle) Delete(ctx context.Context) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Delete") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newDeleteCall() if err != nil { return err @@ -139,7 +152,10 @@ func (b *BucketHandle) Object(name string) *ObjectHandle { } // Attrs returns the metadata for the bucket. -func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { +func (b *BucketHandle) Attrs(ctx context.Context) (attrs *BucketAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Attrs") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newGetCall() if err != nil { return nil, err @@ -155,7 +171,7 @@ func (b *BucketHandle) Attrs(ctx context.Context) (*BucketAttrs, error) { if err != nil { return nil, err } - return newBucket(resp), nil + return newBucket(resp) } func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { @@ -170,17 +186,27 @@ func (b *BucketHandle) newGetCall() (*raw.BucketsGetCall, error) { return req, nil } -func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (*BucketAttrs, error) { +// Update updates a bucket's attributes. +func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) (attrs *BucketAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Create") + defer func() { trace.EndSpan(ctx, err) }() + req, err := b.newPatchCall(&uattrs) if err != nil { return nil, err } + if uattrs.PredefinedACL != "" { + req.PredefinedAcl(uattrs.PredefinedACL) + } + if uattrs.PredefinedDefaultObjectACL != "" { + req.PredefinedDefaultObjectAcl(uattrs.PredefinedDefaultObjectACL) + } // TODO(jba): retry iff metagen is set? rb, err := req.Context(ctx).Do() if err != nil { return nil, err } - return newBucket(rb), nil + return newBucket(rb) } func (b *BucketHandle) newPatchCall(uattrs *BucketAttrsToUpdate) (*raw.BucketsPatchCall, error) { @@ -210,6 +236,24 @@ type BucketAttrs struct { // apply to new objects when no object ACL is provided. DefaultObjectACL []ACLRule + // DefaultEventBasedHold is the default value for event-based hold on + // newly created objects in this bucket. It defaults to false. + DefaultEventBasedHold bool + + // If not empty, applies a predefined set of access controls. It should be set + // only when creating a bucket. + // It is always empty for BucketAttrs returned from the service. + // See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert + // for valid values. + PredefinedACL string + + // If not empty, applies a predefined set of default object access controls. + // It should be set only when creating a bucket. + // It is always empty for BucketAttrs returned from the service. + // See https://cloud.google.com/storage/docs/json_api/v1/buckets/insert + // for valid values. + PredefinedDefaultObjectACL string + // Location is the location of the bucket. It defaults to "US". Location string @@ -245,8 +289,26 @@ type BucketAttrs struct { // Lifecycle is the lifecycle configuration for objects in the bucket. Lifecycle Lifecycle + // Retention policy enforces a minimum retention time for all objects + // contained in the bucket. A RetentionPolicy of nil implies the bucket + // has no minimum data retention. + // + // This feature is in private alpha release. It is not currently available to + // most customers. It might be changed in backwards-incompatible ways and is not + // subject to any SLA or deprecation policy. + RetentionPolicy *RetentionPolicy + // The bucket's Cross-Origin Resource Sharing (CORS) configuration. CORS []CORS + + // The encryption configuration used by default for newly inserted objects. + Encryption *BucketEncryption + + // The logging configuration. + Logging *BucketLogging + + // The website configuration. + Website *BucketWebsite } // Lifecycle is the lifecycle configuration for objects in the bucket. @@ -254,12 +316,42 @@ type Lifecycle struct { Rules []LifecycleRule } +// RetentionPolicy enforces a minimum retention time for all objects +// contained in the bucket. +// +// Any attempt to overwrite or delete objects younger than the retention +// period will result in an error. An unlocked retention policy can be +// modified or removed from the bucket via the Update method. A +// locked retention policy cannot be removed or shortened in duration +// for the lifetime of the bucket. +// +// This feature is in private alpha release. It is not currently available to +// most customers. It might be changed in backwards-incompatible ways and is not +// subject to any SLA or deprecation policy. +type RetentionPolicy struct { + // RetentionPeriod specifies the duration that objects need to be + // retained. Retention duration must be greater than zero and less than + // 100 years. Note that enforcement of retention periods less than a day + // is not guaranteed. Such periods should only be used for testing + // purposes. + RetentionPeriod time.Duration + + // EffectiveTime is the time from which the policy was enforced and + // effective. This field is read-only. + EffectiveTime time.Time + + // IsLocked describes whether the bucket is locked. Once locked, an + // object retention policy cannot be modified. + // This field is read-only. + IsLocked bool +} + const ( // RFC3339 date with only the date segment, used for CreatedBefore in LifecycleRule. rfc3339Date = "2006-01-02" // DeleteAction is a lifecycle action that deletes a live and/or archived - // objects. Takes precendence over SetStorageClass actions. + // objects. Takes precedence over SetStorageClass actions. DeleteAction = "Delete" // SetStorageClassAction changes the storage class of live and/or archived @@ -339,54 +431,65 @@ type LifecycleCondition struct { NumNewerVersions int64 } -func newBucket(b *raw.Bucket) *BucketAttrs { +// BucketLogging holds the bucket's logging configuration, which defines the +// destination bucket and optional name prefix for the current bucket's +// logs. +type BucketLogging struct { + // The destination bucket where the current bucket's logs + // should be placed. + LogBucket string + + // A prefix for log object names. + LogObjectPrefix string +} + +// BucketWebsite holds the bucket's website configuration, controlling how the +// service behaves when accessing bucket contents as a web site. See +// https://cloud.google.com/storage/docs/static-website for more information. +type BucketWebsite struct { + // If the requested object path is missing, the service will ensure the path has + // a trailing '/', append this suffix, and attempt to retrieve the resulting + // object. This allows the creation of index.html objects to represent directory + // pages. + MainPageSuffix string + + // If the requested object path is missing, and any mainPageSuffix object is + // missing, if applicable, the service will return the named object from this + // bucket as the content for a 404 Not Found result. + NotFoundPage string +} + +func newBucket(b *raw.Bucket) (*BucketAttrs, error) { if b == nil { - return nil + return nil, nil } - bucket := &BucketAttrs{ - Name: b.Name, - Location: b.Location, - MetaGeneration: b.Metageneration, - StorageClass: b.StorageClass, - Created: convertTime(b.TimeCreated), - VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled, - Labels: b.Labels, - RequesterPays: b.Billing != nil && b.Billing.RequesterPays, - Lifecycle: toLifecycle(b.Lifecycle), - CORS: toCORS(b.Cors), - } - acl := make([]ACLRule, len(b.Acl)) - for i, rule := range b.Acl { - acl[i] = ACLRule{ - Entity: ACLEntity(rule.Entity), - Role: ACLRole(rule.Role), - } - } - bucket.ACL = acl - objACL := make([]ACLRule, len(b.DefaultObjectAcl)) - for i, rule := range b.DefaultObjectAcl { - objACL[i] = ACLRule{ - Entity: ACLEntity(rule.Entity), - Role: ACLRole(rule.Role), - } + rp, err := toRetentionPolicy(b.RetentionPolicy) + if err != nil { + return nil, err } - bucket.DefaultObjectACL = objACL - return bucket + return &BucketAttrs{ + Name: b.Name, + Location: b.Location, + MetaGeneration: b.Metageneration, + DefaultEventBasedHold: b.DefaultEventBasedHold, + StorageClass: b.StorageClass, + Created: convertTime(b.TimeCreated), + VersioningEnabled: b.Versioning != nil && b.Versioning.Enabled, + ACL: toBucketACLRules(b.Acl), + DefaultObjectACL: toObjectACLRules(b.DefaultObjectAcl), + Labels: b.Labels, + RequesterPays: b.Billing != nil && b.Billing.RequesterPays, + Lifecycle: toLifecycle(b.Lifecycle), + RetentionPolicy: rp, + CORS: toCORS(b.Cors), + Encryption: toBucketEncryption(b.Encryption), + Logging: toBucketLogging(b.Logging), + Website: toBucketWebsite(b.Website), + }, nil } // toRawBucket copies the editable attribute from b to the raw library's Bucket type. func (b *BucketAttrs) toRawBucket() *raw.Bucket { - var acl []*raw.BucketAccessControl - if len(b.ACL) > 0 { - acl = make([]*raw.BucketAccessControl, len(b.ACL)) - for i, rule := range b.ACL { - acl[i] = &raw.BucketAccessControl{ - Entity: string(rule.Entity), - Role: string(rule.Role), - } - } - } - dACL := toRawObjectACL(b.DefaultObjectACL) // Copy label map. var labels map[string]string if len(b.Labels) > 0 { @@ -408,19 +511,23 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket { } return &raw.Bucket{ Name: b.Name, - DefaultObjectAcl: dACL, Location: b.Location, StorageClass: b.StorageClass, - Acl: acl, + Acl: toRawBucketACL(b.ACL), + DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL), Versioning: v, Labels: labels, Billing: bb, Lifecycle: toRawLifecycle(b.Lifecycle), + RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(), Cors: toRawCORS(b.CORS), + Encryption: b.Encryption.toRawBucketEncryption(), + Logging: b.Logging.toRawBucketLogging(), + Website: b.Website.toRawBucketWebsite(), } } -// The bucket's Cross-Origin Resource Sharing (CORS) configuration. +// CORS is the bucket's Cross-Origin Resource Sharing (CORS) configuration. type CORS struct { // MaxAge is the value to return in the Access-Control-Max-Age // header used in preflight responses. @@ -442,18 +549,61 @@ type CORS struct { ResponseHeaders []string } +// BucketEncryption is a bucket's encryption configuration. +type BucketEncryption struct { + // A Cloud KMS key name, in the form + // projects/P/locations/L/keyRings/R/cryptoKeys/K, that will be used to encrypt + // objects inserted into this bucket, if no encryption method is specified. + // The key's location must be the same as the bucket's. + DefaultKMSKeyName string +} + +// BucketAttrsToUpdate define the attributes to update during an Update call. type BucketAttrsToUpdate struct { - // VersioningEnabled, if set, updates whether the bucket uses versioning. + // If set, updates whether the bucket uses versioning. VersioningEnabled optional.Bool - // RequesterPays, if set, updates whether the bucket is a Requester Pays bucket. + // If set, updates whether the bucket is a Requester Pays bucket. RequesterPays optional.Bool - // CORS, if set, replaces the CORS configuration with a new configuration. - // When an empty slice is provided, all CORS policies are removed; when nil - // is provided, the value is ignored in the update. + // DefaultEventBasedHold is the default value for event-based hold on + // newly created objects in this bucket. + DefaultEventBasedHold optional.Bool + + // If set, updates the retention policy of the bucket. Using + // RetentionPolicy.RetentionPeriod = 0 will delete the existing policy. + // + // This feature is in private alpha release. It is not currently available to + // most customers. It might be changed in backwards-incompatible ways and is not + // subject to any SLA or deprecation policy. + RetentionPolicy *RetentionPolicy + + // If set, replaces the CORS configuration with a new configuration. + // An empty (rather than nil) slice causes all CORS policies to be removed. CORS []CORS + // If set, replaces the encryption configuration of the bucket. Using + // BucketEncryption.DefaultKMSKeyName = "" will delete the existing + // configuration. + Encryption *BucketEncryption + + // If set, replaces the lifecycle configuration of the bucket. + Lifecycle *Lifecycle + + // If set, replaces the logging configuration of the bucket. + Logging *BucketLogging + + // If set, replaces the website configuration of the bucket. + Website *BucketWebsite + + // If not empty, applies a predefined set of access controls. + // See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch. + PredefinedACL string + + // If not empty, applies a predefined set of default object access controls. + // See https://cloud.google.com/storage/docs/json_api/v1/buckets/patch. + PredefinedDefaultObjectACL string + setLabels map[string]string deleteLabels map[string]bool } @@ -482,6 +632,18 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { rb.Cors = toRawCORS(ua.CORS) rb.ForceSendFields = append(rb.ForceSendFields, "Cors") } + if ua.DefaultEventBasedHold != nil { + rb.DefaultEventBasedHold = optional.ToBool(ua.DefaultEventBasedHold) + rb.ForceSendFields = append(rb.ForceSendFields, "DefaultEventBasedHold") + } + if ua.RetentionPolicy != nil { + if ua.RetentionPolicy.RetentionPeriod == 0 { + rb.NullFields = append(rb.NullFields, "RetentionPolicy") + rb.RetentionPolicy = nil + } else { + rb.RetentionPolicy = ua.RetentionPolicy.toRawRetentionPolicy() + } + } if ua.VersioningEnabled != nil { rb.Versioning = &raw.BucketVersioning{ Enabled: optional.ToBool(ua.VersioningEnabled), @@ -494,6 +656,43 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { ForceSendFields: []string{"RequesterPays"}, } } + if ua.Encryption != nil { + if ua.Encryption.DefaultKMSKeyName == "" { + rb.NullFields = append(rb.NullFields, "Encryption") + rb.Encryption = nil + } else { + rb.Encryption = ua.Encryption.toRawBucketEncryption() + } + } + if ua.Lifecycle != nil { + rb.Lifecycle = toRawLifecycle(*ua.Lifecycle) + } + if ua.Logging != nil { + if *ua.Logging == (BucketLogging{}) { + rb.NullFields = append(rb.NullFields, "Logging") + rb.Logging = nil + } else { + rb.Logging = ua.Logging.toRawBucketLogging() + } + } + if ua.Website != nil { + if *ua.Website == (BucketWebsite{}) { + rb.NullFields = append(rb.NullFields, "Website") + rb.Website = nil + } else { + rb.Website = ua.Website.toRawBucketWebsite() + } + } + if ua.PredefinedACL != "" { + // Clear ACL or the call will fail. + rb.Acl = nil + rb.ForceSendFields = append(rb.ForceSendFields, "Acl") + } + if ua.PredefinedDefaultObjectACL != "" { + // Clear ACLs or the call will fail. + rb.DefaultObjectAcl = nil + rb.ForceSendFields = append(rb.ForceSendFields, "DefaultObjectAcl") + } if ua.setLabels != nil || ua.deleteLabels != nil { rb.Labels = map[string]string{} for k, v := range ua.setLabels { @@ -511,7 +710,7 @@ func (ua *BucketAttrsToUpdate) toRawBucket() *raw.Bucket { // If returns a new BucketHandle that applies a set of preconditions. // Preconditions already set on the BucketHandle are ignored. -// Operations on the new handle will only occur if the preconditions are +// Operations on the new handle will return an error if the preconditions are not // satisfied. The only valid preconditions for buckets are MetagenerationMatch // and MetagenerationNotMatch. func (b *BucketHandle) If(conds BucketConditions) *BucketHandle { @@ -558,6 +757,25 @@ func (b *BucketHandle) UserProject(projectID string) *BucketHandle { return &b2 } +// LockRetentionPolicy locks a bucket's retention policy until a previously-configured +// RetentionPeriod past the EffectiveTime. Note that if RetentionPeriod is set to less +// than a day, the retention policy is treated as a development configuration and locking +// will have no effect. The BucketHandle must have a metageneration condition that +// matches the bucket's metageneration. See BucketHandle.If. +// +// This feature is in private alpha release. It is not currently available to +// most customers. It might be changed in backwards-incompatible ways and is not +// subject to any SLA or deprecation policy. +func (b *BucketHandle) LockRetentionPolicy(ctx context.Context) error { + var metageneration int64 + if b.conds != nil { + metageneration = b.conds.MetagenerationMatch + } + req := b.c.raw.Buckets.LockRetentionPolicy(b.name, metageneration) + _, err := req.Context(ctx).Do() + return err +} + // applyBucketConds modifies the provided call using the conditions in conds. // call is something that quacks like a *raw.WhateverCall. func applyBucketConds(method string, conds *BucketConditions, call interface{}) error { @@ -581,6 +799,30 @@ func applyBucketConds(method string, conds *BucketConditions, call interface{}) return nil } +func (rp *RetentionPolicy) toRawRetentionPolicy() *raw.BucketRetentionPolicy { + if rp == nil { + return nil + } + return &raw.BucketRetentionPolicy{ + RetentionPeriod: int64(rp.RetentionPeriod / time.Second), + } +} + +func toRetentionPolicy(rp *raw.BucketRetentionPolicy) (*RetentionPolicy, error) { + if rp == nil { + return nil, nil + } + t, err := time.Parse(time.RFC3339, rp.EffectiveTime) + if err != nil { + return nil, err + } + return &RetentionPolicy{ + RetentionPeriod: time.Duration(rp.RetentionPeriod) * time.Second, + EffectiveTime: t, + IsLocked: rp.IsLocked, + }, nil +} + func toRawCORS(c []CORS) []*raw.BucketCors { var out []*raw.BucketCors for _, v := range c { @@ -677,6 +919,62 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { return l } +func (e *BucketEncryption) toRawBucketEncryption() *raw.BucketEncryption { + if e == nil { + return nil + } + return &raw.BucketEncryption{ + DefaultKmsKeyName: e.DefaultKMSKeyName, + } +} + +func toBucketEncryption(e *raw.BucketEncryption) *BucketEncryption { + if e == nil { + return nil + } + return &BucketEncryption{DefaultKMSKeyName: e.DefaultKmsKeyName} +} + +func (b *BucketLogging) toRawBucketLogging() *raw.BucketLogging { + if b == nil { + return nil + } + return &raw.BucketLogging{ + LogBucket: b.LogBucket, + LogObjectPrefix: b.LogObjectPrefix, + } +} + +func toBucketLogging(b *raw.BucketLogging) *BucketLogging { + if b == nil { + return nil + } + return &BucketLogging{ + LogBucket: b.LogBucket, + LogObjectPrefix: b.LogObjectPrefix, + } +} + +func (w *BucketWebsite) toRawBucketWebsite() *raw.BucketWebsite { + if w == nil { + return nil + } + return &raw.BucketWebsite{ + MainPageSuffix: w.MainPageSuffix, + NotFoundPage: w.NotFoundPage, + } +} + +func toBucketWebsite(w *raw.BucketWebsite) *BucketWebsite { + if w == nil { + return nil + } + return &BucketWebsite{ + MainPageSuffix: w.MainPageSuffix, + NotFoundPage: w.NotFoundPage, + } +} + // Objects returns an iterator over the objects in the bucket that match the Query q. // If q is nil, no filtering is done. func (b *BucketHandle) Objects(ctx context.Context, q *Query) *ObjectIterator { @@ -758,8 +1056,6 @@ func (it *ObjectIterator) fetch(pageSize int, pageToken string) (string, error) return resp.NextPageToken, nil } -// TODO(jbd): Add storage.buckets.update. - // Buckets returns an iterator over the buckets in the project. You may // optionally set the iterator's Prefix field to restrict the list to buckets // whose names begin with the prefix. By default, all buckets in the project @@ -805,7 +1101,7 @@ func (it *BucketIterator) Next() (*BucketAttrs, error) { // PageInfo supports pagination. See the google.golang.org/api/iterator package for details. func (it *BucketIterator) PageInfo() *iterator.PageInfo { return it.pageInfo } -func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) { +func (it *BucketIterator) fetch(pageSize int, pageToken string) (token string, err error) { req := it.client.raw.Buckets.List(it.projectID) setClientHeader(req.Header()) req.Projection("full") @@ -815,7 +1111,6 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) req.MaxResults(int64(pageSize)) } var resp *raw.Buckets - var err error err = runWithRetry(it.ctx, func() error { resp, err = req.Context(it.ctx).Do() return err @@ -824,7 +1119,11 @@ func (it *BucketIterator) fetch(pageSize int, pageToken string) (string, error) return "", err } for _, item := range resp.Items { - it.buckets = append(it.buckets, newBucket(item)) + b, err := newBucket(item) + if err != nil { + return "", err + } + it.buckets = append(it.buckets, b) } return resp.NextPageToken, nil } diff --git a/vendor/cloud.google.com/go/storage/copy.go b/vendor/cloud.google.com/go/storage/copy.go index d0a999c1b..52162e72d 100644 --- a/vendor/cloud.google.com/go/storage/copy.go +++ b/vendor/cloud.google.com/go/storage/copy.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,10 +15,11 @@ package storage import ( + "context" "errors" "fmt" - "golang.org/x/net/context" + "cloud.google.com/go/internal/trace" raw "google.golang.org/api/storage/v1" ) @@ -59,17 +60,32 @@ type Copier struct { // ProgressFunc should return quickly without blocking. ProgressFunc func(copiedBytes, totalBytes uint64) + // The Cloud KMS key, in the form projects/P/locations/L/keyRings/R/cryptoKeys/K, + // that will be used to encrypt the object. Overrides the object's KMSKeyName, if + // any. + // + // Providing both a DestinationKMSKeyName and a customer-supplied encryption key + // (via ObjectHandle.Key) on the destination object will result in an error when + // Run is called. + DestinationKMSKeyName string + dst, src *ObjectHandle } // Run performs the copy. -func (c *Copier) Run(ctx context.Context) (*ObjectAttrs, error) { +func (c *Copier) Run(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Copier.Run") + defer func() { trace.EndSpan(ctx, err) }() + if err := c.src.validate(); err != nil { return nil, err } if err := c.dst.validate(); err != nil { return nil, err } + if c.DestinationKMSKeyName != "" && c.dst.encryptionKey != nil { + return nil, errors.New("storage: cannot use DestinationKMSKeyName with a customer-supplied encryption key") + } // Convert destination attributes to raw form, omitting the bucket. // If the bucket is included but name or content-type aren't, the service // returns a 400 with "Required" as the only message. Omitting the bucket @@ -96,6 +112,12 @@ func (c *Copier) callRewrite(ctx context.Context, rawObj *raw.Object) (*raw.Rewr if c.RewriteToken != "" { call.RewriteToken(c.RewriteToken) } + if c.DestinationKMSKeyName != "" { + call.DestinationKmsKeyName(c.DestinationKMSKeyName) + } + if c.PredefinedACL != "" { + call.DestinationPredefinedAcl(c.PredefinedACL) + } if err := applyConds("Copy destination", c.dst.gen, c.dst.conds, call); err != nil { return nil, err } @@ -149,7 +171,10 @@ type Composer struct { } // Run performs the compose operation. -func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { +func (c *Composer) Run(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Composer.Run") + defer func() { trace.EndSpan(ctx, err) }() + if err := c.dst.validate(); err != nil { return nil, err } @@ -187,11 +212,13 @@ func (c *Composer) Run(ctx context.Context) (*ObjectAttrs, error) { if c.dst.userProject != "" { call.UserProject(c.dst.userProject) } + if c.PredefinedACL != "" { + call.DestinationPredefinedAcl(c.PredefinedACL) + } if err := setEncryptionHeaders(call.Header(), c.dst.encryptionKey, false); err != nil { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if err != nil { diff --git a/vendor/cloud.google.com/go/storage/doc.go b/vendor/cloud.google.com/go/storage/doc.go index 1f32fde95..e277f1d95 100644 --- a/vendor/cloud.google.com/go/storage/doc.go +++ b/vendor/cloud.google.com/go/storage/doc.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,9 +19,14 @@ Google Cloud Storage stores data in named objects, which are grouped into bucket More information about Google Cloud Storage is available at https://cloud.google.com/storage/docs. -All of the methods of this package use exponential backoff to retry calls -that fail with certain errors, as described in -https://cloud.google.com/storage/docs/exponential-backoff. +See https://godoc.org/cloud.google.com/go for authentication, timeouts, +connection pooling and similar aspects of this package. + +All of the methods of this package use exponential backoff to retry calls that fail +with certain errors, as described in +https://cloud.google.com/storage/docs/exponential-backoff. Retrying continues +indefinitely unless the controlling context is canceled or the client is closed. See +context.WithTimeout and context.WithCancel. Creating a Client @@ -61,7 +66,7 @@ global across all projects. Each bucket has associated metadata, represented in this package by BucketAttrs. The third argument to BucketHandle.Create allows you to set -the intial BucketAttrs of a bucket. To retrieve a bucket's attributes, use +the initial BucketAttrs of a bucket. To retrieve a bucket's attributes, use Attrs: attrs, err := bkt.Attrs(ctx) @@ -74,15 +79,16 @@ Attrs: Objects An object holds arbitrary data as a sequence of bytes, like a file. You -refer to objects using a handle, just as with buckets. You can use the -standard Go io.Reader and io.Writer interfaces to read and write -object data: +refer to objects using a handle, just as with buckets, but unlike buckets +you don't explicitly create an object. Instead, the first time you write +to an object it will be created. You can use the standard Go io.Reader +and io.Writer interfaces to read and write object data: obj := bkt.Object("data") // Write something to obj. // w implements io.Writer. w := obj.NewWriter(ctx) - // Write some text to obj. This will overwrite whatever is there. + // Write some text to obj. This will either create the object or overwrite whatever is there already. if _, err := fmt.Fprintf(w, "This object contains text.\n"); err != nil { // TODO: Handle error. } @@ -158,9 +164,13 @@ SignedURL for details. } fmt.Println(url) -Authentication +Errors + +Errors returned by this client are often of the type [`googleapi.Error`](https://godoc.org/google.golang.org/api/googleapi#Error). +These errors can be introspected for more information by type asserting to the richer `googleapi.Error` type. For example: -See examples of authorization and authentication at -https://godoc.org/cloud.google.com/go#pkg-examples. + if e, ok := err.(*googleapi.Error); ok { + if e.Code = 409 { ... } + } */ package storage // import "cloud.google.com/go/storage" diff --git a/vendor/cloud.google.com/go/storage/go110.go b/vendor/cloud.google.com/go/storage/go110.go index b85e8c3b9..206813f0c 100644 --- a/vendor/cloud.google.com/go/storage/go110.go +++ b/vendor/cloud.google.com/go/storage/go110.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +24,8 @@ func shouldRetry(err error) bool { // Retry on 429 and 5xx, according to // https://cloud.google.com/storage/docs/exponential-backoff. return e.Code == 429 || (e.Code >= 500 && e.Code < 600) + case interface{ Temporary() bool }: + return e.Temporary() default: return false } diff --git a/vendor/cloud.google.com/go/storage/go17.go b/vendor/cloud.google.com/go/storage/go17.go deleted file mode 100644 index 982db4e1a..000000000 --- a/vendor/cloud.google.com/go/storage/go17.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build go1.7 - -package storage - -import ( - "context" - "net/http" -) - -func withContext(r *http.Request, ctx context.Context) *http.Request { - return r.WithContext(ctx) -} diff --git a/vendor/cloud.google.com/go/storage/iam.go b/vendor/cloud.google.com/go/storage/iam.go index 9365509ed..9d9360671 100644 --- a/vendor/cloud.google.com/go/storage/iam.go +++ b/vendor/cloud.google.com/go/storage/iam.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,8 +15,10 @@ package storage import ( + "context" + "cloud.google.com/go/iam" - "golang.org/x/net/context" + "cloud.google.com/go/internal/trace" raw "google.golang.org/api/storage/v1" iampb "google.golang.org/genproto/googleapis/iam/v1" ) @@ -35,14 +37,16 @@ type iamClient struct { userProject string } -func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, error) { +func (c *iamClient) Get(ctx context.Context, resource string) (p *iampb.Policy, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Get") + defer func() { trace.EndSpan(ctx, err) }() + call := c.raw.Buckets.GetIamPolicy(resource) setClientHeader(call.Header()) if c.userProject != "" { call.UserProject(c.userProject) } var rp *raw.Policy - var err error err = runWithRetry(ctx, func() error { rp, err = call.Context(ctx).Do() return err @@ -53,7 +57,10 @@ func (c *iamClient) Get(ctx context.Context, resource string) (*iampb.Policy, er return iamFromStoragePolicy(rp), nil } -func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) error { +func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Set") + defer func() { trace.EndSpan(ctx, err) }() + rp := iamToStoragePolicy(p) call := c.raw.Buckets.SetIamPolicy(resource, rp) setClientHeader(call.Header()) @@ -66,14 +73,16 @@ func (c *iamClient) Set(ctx context.Context, resource string, p *iampb.Policy) e }) } -func (c *iamClient) Test(ctx context.Context, resource string, perms []string) ([]string, error) { +func (c *iamClient) Test(ctx context.Context, resource string, perms []string) (permissions []string, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.IAM.Test") + defer func() { trace.EndSpan(ctx, err) }() + call := c.raw.Buckets.TestIamPermissions(resource, perms) setClientHeader(call.Header()) if c.userProject != "" { call.UserProject(c.userProject) } var res *raw.TestIamPermissionsResponse - var err error err = runWithRetry(ctx, func() error { res, err = call.Context(ctx).Do() return err diff --git a/vendor/cloud.google.com/go/storage/invoke.go b/vendor/cloud.google.com/go/storage/invoke.go index 46423a8b2..0a03f1101 100644 --- a/vendor/cloud.google.com/go/storage/invoke.go +++ b/vendor/cloud.google.com/go/storage/invoke.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,9 +15,10 @@ package storage import ( + "context" + "cloud.google.com/go/internal" gax "github.com/googleapis/gax-go" - "golang.org/x/net/context" ) // runWithRetry calls the function until it returns nil or a non-retryable error, or diff --git a/vendor/cloud.google.com/go/storage/not_go110.go b/vendor/cloud.google.com/go/storage/not_go110.go index c354e74bf..66fa45bea 100644 --- a/vendor/cloud.google.com/go/storage/not_go110.go +++ b/vendor/cloud.google.com/go/storage/not_go110.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -34,6 +34,8 @@ func shouldRetry(err error) bool { // Unfortunately the error type is unexported, so we resort to string // matching. return strings.Contains(e.Error(), "REFUSED_STREAM") + case interface{ Temporary() bool }: + return e.Temporary() default: return false } diff --git a/vendor/cloud.google.com/go/storage/not_go17.go b/vendor/cloud.google.com/go/storage/not_go17.go deleted file mode 100644 index 1f6f7ae95..000000000 --- a/vendor/cloud.google.com/go/storage/not_go17.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2017 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !go1.7 - -package storage - -import ( - "net/http" -) - -func withContext(r *http.Request, _ interface{}) *http.Request { - // In Go 1.6 and below, ignore the context. - return r -} diff --git a/vendor/cloud.google.com/go/storage/notifications.go b/vendor/cloud.google.com/go/storage/notifications.go index b95dd453a..84619b6d5 100644 --- a/vendor/cloud.google.com/go/storage/notifications.go +++ b/vendor/cloud.google.com/go/storage/notifications.go @@ -1,4 +1,4 @@ -// Copyright 2017 Google Inc. All Rights Reserved. +// Copyright 2017 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,11 +15,12 @@ package storage import ( + "context" "errors" "fmt" "regexp" - "golang.org/x/net/context" + "cloud.google.com/go/internal/trace" raw "google.golang.org/api/storage/v1" ) @@ -118,7 +119,10 @@ func toRawNotification(n *Notification) *raw.Notification { // AddNotification adds a notification to b. You must set n's TopicProjectID, TopicID // and PayloadFormat, and must not set its ID. The other fields are all optional. The // returned Notification's ID can be used to refer to it. -func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (*Notification, error) { +func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (ret *Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.AddNotification") + defer func() { trace.EndSpan(ctx, err) }() + if n.ID != "" { return nil, errors.New("storage: AddNotification: ID must not be set") } @@ -142,14 +146,16 @@ func (b *BucketHandle) AddNotification(ctx context.Context, n *Notification) (*N // Notifications returns all the Notifications configured for this bucket, as a map // indexed by notification ID. -func (b *BucketHandle) Notifications(ctx context.Context) (map[string]*Notification, error) { +func (b *BucketHandle) Notifications(ctx context.Context) (n map[string]*Notification, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.Notifications") + defer func() { trace.EndSpan(ctx, err) }() + call := b.c.raw.Notifications.List(b.name) setClientHeader(call.Header()) if b.userProject != "" { call.UserProject(b.userProject) } var res *raw.Notifications - var err error err = runWithRetry(ctx, func() error { res, err = call.Context(ctx).Do() return err @@ -169,7 +175,10 @@ func notificationsToMap(rns []*raw.Notification) map[string]*Notification { } // DeleteNotification deletes the notification with the given ID. -func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) error { +func (b *BucketHandle) DeleteNotification(ctx context.Context, id string) (err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Bucket.DeleteNotification") + defer func() { trace.EndSpan(ctx, err) }() + call := b.c.raw.Notifications.Delete(b.name, id) setClientHeader(call.Header()) if b.userProject != "" { diff --git a/vendor/cloud.google.com/go/storage/reader.go b/vendor/cloud.google.com/go/storage/reader.go index 68b3c3bf4..50f381f91 100644 --- a/vendor/cloud.google.com/go/storage/reader.go +++ b/vendor/cloud.google.com/go/storage/reader.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All Rights Reserved. +// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,13 +15,263 @@ package storage import ( + "context" + "errors" "fmt" "hash/crc32" "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strconv" + "strings" + "time" + + "cloud.google.com/go/internal/trace" + "google.golang.org/api/googleapi" ) var crc32cTable = crc32.MakeTable(crc32.Castagnoli) +// ReaderObjectAttrs are attributes about the object being read. These are populated +// during the New call. This struct only holds a subset of object attributes: to +// get the full set of attributes, use ObjectHandle.Attrs. +// +// Each field is read-only. +type ReaderObjectAttrs struct { + // Size is the length of the object's content. + Size int64 + + // ContentType is the MIME type of the object's content. + ContentType string + + // ContentEncoding is the encoding of the object's content. + ContentEncoding string + + // CacheControl specifies whether and for how long browser and Internet + // caches are allowed to cache your objects. + CacheControl string + + // LastModified is the time that the object was last modified. + LastModified time.Time + + // Generation is the generation number of the object's content. + Generation int64 + + // Metageneration is the version of the metadata for this object at + // this generation. This field is used for preconditions and for + // detecting changes in metadata. A metageneration number is only + // meaningful in the context of a particular generation of a + // particular object. + Metageneration int64 +} + +// NewReader creates a new Reader to read the contents of the +// object. +// ErrObjectNotExist will be returned if the object is not found. +// +// The caller must call Close on the returned Reader when done reading. +func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { + return o.NewRangeReader(ctx, 0, -1) +} + +// NewRangeReader reads part of an object, reading at most length bytes +// starting at the given offset. If length is negative, the object is read +// until the end. +func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (r *Reader, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.NewRangeReader") + defer func() { trace.EndSpan(ctx, err) }() + + if err := o.validate(); err != nil { + return nil, err + } + if offset < 0 { + return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) + } + if o.conds != nil { + if err := o.conds.validate("NewRangeReader"); err != nil { + return nil, err + } + } + u := &url.URL{ + Scheme: "https", + Host: "storage.googleapis.com", + Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), + } + verb := "GET" + if length == 0 { + verb = "HEAD" + } + req, err := http.NewRequest(verb, u.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if o.userProject != "" { + req.Header.Set("X-Goog-User-Project", o.userProject) + } + if o.readCompressed { + req.Header.Set("Accept-Encoding", "gzip") + } + if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { + return nil, err + } + + gen := o.gen + + // Define a function that initiates a Read with offset and length, assuming we + // have already read seen bytes. + reopen := func(seen int64) (*http.Response, error) { + start := offset + seen + if length < 0 && start > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", start)) + } else if length > 0 { + // The end character isn't affected by how many bytes we've seen. + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, offset+length-1)) + } + // We wait to assign conditions here because the generation number can change in between reopen() runs. + req.URL.RawQuery = conditionsQuery(gen, o.conds) + var res *http.Response + err = runWithRetry(ctx, func() error { + res, err = o.c.hc.Do(req) + if err != nil { + return err + } + if res.StatusCode == http.StatusNotFound { + res.Body.Close() + return ErrObjectNotExist + } + if res.StatusCode < 200 || res.StatusCode > 299 { + body, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + return &googleapi.Error{ + Code: res.StatusCode, + Header: res.Header, + Body: string(body), + } + } + if start > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { + res.Body.Close() + return errors.New("storage: partial request not satisfied") + } + // If a generation hasn't been specified, and this is the first response we get, let's record the + // generation. In future requests we'll use this generation as a precondition to avoid data races. + if gen < 0 && res.Header.Get("X-Goog-Generation") != "" { + gen64, err := strconv.ParseInt(res.Header.Get("X-Goog-Generation"), 10, 64) + if err != nil { + return err + } + gen = gen64 + } + return nil + }) + if err != nil { + return nil, err + } + return res, nil + } + + res, err := reopen(0) + if err != nil { + return nil, err + } + var ( + size int64 // total size of object, even if a range was requested. + checkCRC bool + crc uint32 + ) + if res.StatusCode == http.StatusPartialContent { + cr := strings.TrimSpace(res.Header.Get("Content-Range")) + if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { + + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) + if err != nil { + return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) + } + } else { + size = res.ContentLength + // Check the CRC iff all of the following hold: + // - We asked for content (length != 0). + // - We got all the content (status != PartialContent). + // - The server sent a CRC header. + // - The Go http stack did not uncompress the file. + // - We were not served compressed data that was uncompressed on download. + // The problem with the last two cases is that the CRC will not match -- GCS + // computes it on the compressed contents, but we compute it on the + // uncompressed contents. + if length != 0 && !res.Uncompressed && !uncompressedByServer(res) { + crc, checkCRC = parseCRC32c(res) + } + } + + remain := res.ContentLength + body := res.Body + if length == 0 { + remain = 0 + body.Close() + body = emptyBody + } + var metaGen int64 + if res.Header.Get("X-Goog-Generation") != "" { + metaGen, err = strconv.ParseInt(res.Header.Get("X-Goog-Metageneration"), 10, 64) + if err != nil { + return nil, err + } + } + + var lm time.Time + if res.Header.Get("Last-Modified") != "" { + lm, err = http.ParseTime(res.Header.Get("Last-Modified")) + if err != nil { + return nil, err + } + } + + attrs := ReaderObjectAttrs{ + Size: size, + ContentType: res.Header.Get("Content-Type"), + ContentEncoding: res.Header.Get("Content-Encoding"), + CacheControl: res.Header.Get("Cache-Control"), + LastModified: lm, + Generation: gen, + Metageneration: metaGen, + } + return &Reader{ + Attrs: attrs, + body: body, + size: size, + remain: remain, + wantCRC: crc, + checkCRC: checkCRC, + reopen: reopen, + }, nil +} + +func uncompressedByServer(res *http.Response) bool { + // If the data is stored as gzip but is not encoded as gzip, then it + // was uncompressed by the server. + return res.Header.Get("X-Goog-Stored-Content-Encoding") == "gzip" && + res.Header.Get("Content-Encoding") != "gzip" +} + +func parseCRC32c(res *http.Response) (uint32, bool) { + const prefix = "crc32c=" + for _, spec := range res.Header["X-Goog-Hash"] { + if strings.HasPrefix(spec, prefix) { + c, err := decodeUint32(spec[len(prefix):]) + if err == nil { + return c, true + } + } + } + return 0, false +} + +var emptyBody = ioutil.NopCloser(strings.NewReader("")) + // Reader reads a Cloud Storage object. // It implements io.Reader. // @@ -29,15 +279,13 @@ var crc32cTable = crc32.MakeTable(crc32.Castagnoli) // the stored CRC, returning an error from Read if there is a mismatch. This integrity check // is skipped if transcoding occurs. See https://cloud.google.com/storage/docs/transcoding. type Reader struct { - body io.ReadCloser - remain, size int64 - contentType string - contentEncoding string - cacheControl string - checkCRC bool // should we check the CRC? - wantCRC uint32 // the CRC32c value the server sent in the header - gotCRC uint32 // running crc - checkedCRC bool // did we check the CRC? (For tests.) + Attrs ReaderObjectAttrs + body io.ReadCloser + seen, remain, size int64 + checkCRC bool // should we check the CRC? + wantCRC uint32 // the CRC32c value the server sent in the header + gotCRC uint32 // running crc + reopen func(seen int64) (*http.Response, error) } // Close closes the Reader. It must be called when done reading. @@ -46,7 +294,7 @@ func (r *Reader) Close() error { } func (r *Reader) Read(p []byte) (int, error) { - n, err := r.body.Read(p) + n, err := r.readWithRetry(p) if r.remain != -1 { r.remain -= int64(n) } @@ -55,8 +303,7 @@ func (r *Reader) Read(p []byte) (int, error) { // Check CRC here. It would be natural to check it in Close, but // everybody defers Close on the assumption that it doesn't return // anything worth looking at. - if r.remain == 0 { // Only check if we have Content-Length. - r.checkedCRC = true + if err == io.EOF { if r.gotCRC != r.wantCRC { return n, fmt.Errorf("storage: bad CRC on read: got %d, want %d", r.gotCRC, r.wantCRC) @@ -66,11 +313,42 @@ func (r *Reader) Read(p []byte) (int, error) { return n, err } +func (r *Reader) readWithRetry(p []byte) (int, error) { + n := 0 + for len(p[n:]) > 0 { + m, err := r.body.Read(p[n:]) + n += m + r.seen += int64(m) + if !shouldRetryRead(err) { + return n, err + } + // Read failed, but we will try again. Send a ranged read request that takes + // into account the number of bytes we've already seen. + res, err := r.reopen(r.seen) + if err != nil { + // reopen already retries + return n, err + } + r.body.Close() + r.body = res.Body + } + return n, nil +} + +func shouldRetryRead(err error) bool { + if err == nil { + return false + } + return strings.HasSuffix(err.Error(), "INTERNAL_ERROR") && strings.Contains(reflect.TypeOf(err).String(), "http2") +} + // Size returns the size of the object in bytes. // The returned value is always the same and is not affected by // calls to Read or Close. +// +// Deprecated: use Reader.Attrs.Size. func (r *Reader) Size() int64 { - return r.size + return r.Attrs.Size } // Remain returns the number of bytes left to read, or -1 if unknown. @@ -79,16 +357,29 @@ func (r *Reader) Remain() int64 { } // ContentType returns the content type of the object. +// +// Deprecated: use Reader.Attrs.ContentType. func (r *Reader) ContentType() string { - return r.contentType + return r.Attrs.ContentType } // ContentEncoding returns the content encoding of the object. +// +// Deprecated: use Reader.Attrs.ContentEncoding. func (r *Reader) ContentEncoding() string { - return r.contentEncoding + return r.Attrs.ContentEncoding } // CacheControl returns the cache control of the object. +// +// Deprecated: use Reader.Attrs.CacheControl. func (r *Reader) CacheControl() string { - return r.cacheControl + return r.Attrs.CacheControl +} + +// LastModified returns the value of the Last-Modified header. +// +// Deprecated: use Reader.Attrs.LastModified. +func (r *Reader) LastModified() (time.Time, error) { + return r.Attrs.LastModified, nil } diff --git a/vendor/cloud.google.com/go/storage/storage.go b/vendor/cloud.google.com/go/storage/storage.go index 0723f07ef..6c62e3e1d 100644 --- a/vendor/cloud.google.com/go/storage/storage.go +++ b/vendor/cloud.google.com/go/storage/storage.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package storage import ( "bytes" + "context" "crypto" "crypto/rand" "crypto/rsa" @@ -26,7 +27,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "net/url" "reflect" @@ -37,18 +37,19 @@ import ( "time" "unicode/utf8" - "google.golang.org/api/option" - htransport "google.golang.org/api/transport/http" - "cloud.google.com/go/internal/optional" + "cloud.google.com/go/internal/trace" "cloud.google.com/go/internal/version" - "golang.org/x/net/context" "google.golang.org/api/googleapi" + "google.golang.org/api/option" raw "google.golang.org/api/storage/v1" + htransport "google.golang.org/api/transport/http" ) var ( + // ErrBucketNotExist indicates that the bucket does not exist. ErrBucketNotExist = errors.New("storage: bucket doesn't exist") + // ErrObjectNotExist indicates that the object does not exist. ErrObjectNotExist = errors.New("storage: object doesn't exist") ) @@ -112,8 +113,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*Client, error // // Close need not be called at program exit. func (c *Client) Close() error { - // Set fields to nil so that subsequent uses - // will panic. + // Set fields to nil so that subsequent uses will panic. c.hc = nil c.raw = nil return nil @@ -346,7 +346,7 @@ func (o *ObjectHandle) Generation(gen int64) *ObjectHandle { // If returns a new ObjectHandle that applies a set of preconditions. // Preconditions already set on the ObjectHandle are ignored. -// Operations on the new handle will only occur if the preconditions are +// Operations on the new handle will return an error if the preconditions are not // satisfied. See https://cloud.google.com/storage/docs/generations-preconditions // for more details. func (o *ObjectHandle) If(conds Conditions) *ObjectHandle { @@ -368,7 +368,10 @@ func (o *ObjectHandle) Key(encryptionKey []byte) *ObjectHandle { // Attrs returns meta information about the object. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { +func (o *ObjectHandle) Attrs(ctx context.Context) (attrs *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Attrs") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -383,7 +386,6 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -398,7 +400,10 @@ func (o *ObjectHandle) Attrs(ctx context.Context) (*ObjectAttrs, error) { // Update updates an object with the provided attributes. // All zero-value attributes are ignored. // ErrObjectNotExist will be returned if the object is not found. -func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (*ObjectAttrs, error) { +func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) (oa *ObjectAttrs, err error) { + ctx = trace.StartSpan(ctx, "cloud.google.com/go/storage.Object.Update") + defer func() { trace.EndSpan(ctx, err) }() + if err := o.validate(); err != nil { return nil, err } @@ -437,6 +442,14 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( attrs.CacheControl = optional.ToString(uattrs.CacheControl) forceSendFields = append(forceSendFields, "CacheControl") } + if uattrs.EventBasedHold != nil { + attrs.EventBasedHold = optional.ToBool(uattrs.EventBasedHold) + forceSendFields = append(forceSendFields, "EventBasedHold") + } + if uattrs.TemporaryHold != nil { + attrs.TemporaryHold = optional.ToBool(uattrs.TemporaryHold) + forceSendFields = append(forceSendFields, "TemporaryHold") + } if uattrs.Metadata != nil { attrs.Metadata = uattrs.Metadata if len(attrs.Metadata) == 0 { @@ -462,11 +475,13 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( if o.userProject != "" { call.UserProject(o.userProject) } + if uattrs.PredefinedACL != "" { + call.PredefinedAcl(uattrs.PredefinedACL) + } if err := setEncryptionHeaders(call.Header(), o.encryptionKey, false); err != nil { return nil, err } var obj *raw.Object - var err error setClientHeader(call.Header()) err = runWithRetry(ctx, func() error { obj, err = call.Do(); return err }) if e, ok := err.(*googleapi.Error); ok && e.Code == http.StatusNotFound { @@ -478,6 +493,16 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( return newObject(obj), nil } +// BucketName returns the name of the bucket. +func (o *ObjectHandle) BucketName() string { + return o.bucket +} + +// ObjectName returns the name of the object. +func (o *ObjectHandle) ObjectName() string { + return o.object +} + // ObjectAttrsToUpdate is used to update the attributes of an object. // Only fields set to non-nil values will be updated. // Set a field to its zero value to delete it. @@ -490,6 +515,8 @@ func (o *ObjectHandle) Update(ctx context.Context, uattrs ObjectAttrsToUpdate) ( // Metadata: map[string]string{}, // } type ObjectAttrsToUpdate struct { + EventBasedHold optional.Bool + TemporaryHold optional.Bool ContentType optional.String ContentLanguage optional.String ContentEncoding optional.String @@ -497,6 +524,10 @@ type ObjectAttrsToUpdate struct { CacheControl optional.String Metadata map[string]string // set to map[string]string{} to delete ACL []ACLRule + + // If not empty, applies a predefined set of access controls. ACL must be nil. + // See https://cloud.google.com/storage/docs/json_api/v1/objects/patch. + PredefinedACL string } // Delete deletes the single specified object. @@ -532,144 +563,6 @@ func (o *ObjectHandle) ReadCompressed(compressed bool) *ObjectHandle { return &o2 } -// NewReader creates a new Reader to read the contents of the -// object. -// ErrObjectNotExist will be returned if the object is not found. -// -// The caller must call Close on the returned Reader when done reading. -func (o *ObjectHandle) NewReader(ctx context.Context) (*Reader, error) { - return o.NewRangeReader(ctx, 0, -1) -} - -// NewRangeReader reads part of an object, reading at most length bytes -// starting at the given offset. If length is negative, the object is read -// until the end. -func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64) (*Reader, error) { - if err := o.validate(); err != nil { - return nil, err - } - if offset < 0 { - return nil, fmt.Errorf("storage: invalid offset %d < 0", offset) - } - if o.conds != nil { - if err := o.conds.validate("NewRangeReader"); err != nil { - return nil, err - } - } - u := &url.URL{ - Scheme: "https", - Host: "storage.googleapis.com", - Path: fmt.Sprintf("/%s/%s", o.bucket, o.object), - RawQuery: conditionsQuery(o.gen, o.conds), - } - verb := "GET" - if length == 0 { - verb = "HEAD" - } - req, err := http.NewRequest(verb, u.String(), nil) - if err != nil { - return nil, err - } - req = withContext(req, ctx) - if length < 0 && offset > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } else if length > 0 { - req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } - if o.userProject != "" { - req.Header.Set("X-Goog-User-Project", o.userProject) - } - if o.readCompressed { - req.Header.Set("Accept-Encoding", "gzip") - } - if err := setEncryptionHeaders(req.Header, o.encryptionKey, false); err != nil { - return nil, err - } - var res *http.Response - err = runWithRetry(ctx, func() error { - res, err = o.c.hc.Do(req) - if err != nil { - return err - } - if res.StatusCode == http.StatusNotFound { - res.Body.Close() - return ErrObjectNotExist - } - if res.StatusCode < 200 || res.StatusCode > 299 { - body, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - return &googleapi.Error{ - Code: res.StatusCode, - Header: res.Header, - Body: string(body), - } - } - if offset > 0 && length != 0 && res.StatusCode != http.StatusPartialContent { - res.Body.Close() - return errors.New("storage: partial request not satisfied") - } - return nil - }) - if err != nil { - return nil, err - } - - var size int64 // total size of object, even if a range was requested. - if res.StatusCode == http.StatusPartialContent { - cr := strings.TrimSpace(res.Header.Get("Content-Range")) - if !strings.HasPrefix(cr, "bytes ") || !strings.Contains(cr, "/") { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - size, err = strconv.ParseInt(cr[strings.LastIndex(cr, "/")+1:], 10, 64) - if err != nil { - return nil, fmt.Errorf("storage: invalid Content-Range %q", cr) - } - } else { - size = res.ContentLength - } - - remain := res.ContentLength - body := res.Body - if length == 0 { - remain = 0 - body.Close() - body = emptyBody - } - var ( - checkCRC bool - crc uint32 - ) - // Even if there is a CRC header, we can't compute the hash on partial data. - if remain == size { - crc, checkCRC = parseCRC32c(res) - } - return &Reader{ - body: body, - size: size, - remain: remain, - contentType: res.Header.Get("Content-Type"), - contentEncoding: res.Header.Get("Content-Encoding"), - cacheControl: res.Header.Get("Cache-Control"), - wantCRC: crc, - checkCRC: checkCRC, - }, nil -} - -func parseCRC32c(res *http.Response) (uint32, bool) { - const prefix = "crc32c=" - for _, spec := range res.Header["X-Goog-Hash"] { - if strings.HasPrefix(spec, prefix) { - c, err := decodeUint32(spec[len(prefix):]) - if err == nil { - return c, true - } - } - } - return 0, false -} - -var emptyBody = ioutil.NopCloser(strings.NewReader("")) - // NewWriter returns a storage Writer that writes to the GCS object // associated with this ObjectHandle. // @@ -683,7 +576,8 @@ var emptyBody = ioutil.NopCloser(strings.NewReader("")) // attribute is specified, the content type will be automatically sniffed // using net/http.DetectContentType. // -// It is the caller's responsibility to call Close when writing is done. +// It is the caller's responsibility to call Close when writing is done. To +// stop writing without saving the data, cancel the context. func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer { return &Writer{ ctx: ctx, @@ -729,34 +623,26 @@ func parseKey(key []byte) (*rsa.PrivateKey, error) { return parsed, nil } -func toRawObjectACL(oldACL []ACLRule) []*raw.ObjectAccessControl { - var acl []*raw.ObjectAccessControl - if len(oldACL) > 0 { - acl = make([]*raw.ObjectAccessControl, len(oldACL)) - for i, rule := range oldACL { - acl[i] = &raw.ObjectAccessControl{ - Entity: string(rule.Entity), - Role: string(rule.Role), - } - } - } - return acl -} - // toRawObject copies the editable attributes from o to the raw library's Object type. func (o *ObjectAttrs) toRawObject(bucket string) *raw.Object { - acl := toRawObjectACL(o.ACL) + var ret string + if !o.RetentionExpirationTime.IsZero() { + ret = o.RetentionExpirationTime.Format(time.RFC3339) + } return &raw.Object{ - Bucket: bucket, - Name: o.Name, - ContentType: o.ContentType, - ContentEncoding: o.ContentEncoding, - ContentLanguage: o.ContentLanguage, - CacheControl: o.CacheControl, - ContentDisposition: o.ContentDisposition, - StorageClass: o.StorageClass, - Acl: acl, - Metadata: o.Metadata, + Bucket: bucket, + Name: o.Name, + EventBasedHold: o.EventBasedHold, + TemporaryHold: o.TemporaryHold, + RetentionExpirationTime: ret, + ContentType: o.ContentType, + ContentEncoding: o.ContentEncoding, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + ContentDisposition: o.ContentDisposition, + StorageClass: o.StorageClass, + Acl: toRawObjectACL(o.ACL), + Metadata: o.Metadata, } } @@ -780,9 +666,32 @@ type ObjectAttrs struct { // headers when serving the object data. CacheControl string + // EventBasedHold specifies whether an object is under event-based hold. New + // objects created in a bucket whose DefaultEventBasedHold is set will + // default to that value. + EventBasedHold bool + + // TemporaryHold specifies whether an object is under temporary hold. While + // this flag is set to true, the object is protected against deletion and + // overwrites. + TemporaryHold bool + + // RetentionExpirationTime is a server-determined value that specifies the + // earliest time that the object's retention period expires. + // This is a read-only field. + RetentionExpirationTime time.Time + // ACL is the list of access control rules for the object. ACL []ACLRule + // If not empty, applies a predefined set of access controls. It should be set + // only when writing, copying or composing an object. When copying or composing, + // it acts as the destinationPredefinedAcl parameter. + // PredefinedACL is always empty for ObjectAttrs returned from the service. + // See https://cloud.google.com/storage/docs/json_api/v1/objects/insert + // for valid values. + PredefinedACL string + // Owner is the owner of the object. This field is read-only. // // If non-zero, it is in the form of "user-". @@ -856,6 +765,14 @@ type ObjectAttrs struct { // encryption in Google Cloud Storage. CustomerKeySHA256 string + // Cloud KMS key name, in the form + // projects/P/locations/L/keyRings/R/cryptoKeys/K, used to encrypt this object, + // if the object is encrypted by such a key. + // + // Providing both a KMSKeyName and a customer-supplied encryption key (via + // ObjectHandle.Key) will result in an error when writing an object. + KMSKeyName string + // Prefix is set only for ObjectAttrs which represent synthetic "directory // entries" when iterating over buckets using Query.Delimiter. See // ObjectIterator.Next. When set, no other fields in ObjectAttrs will be @@ -877,13 +794,6 @@ func newObject(o *raw.Object) *ObjectAttrs { if o == nil { return nil } - acl := make([]ACLRule, len(o.Acl)) - for i, rule := range o.Acl { - acl[i] = ACLRule{ - Entity: ACLEntity(rule.Entity), - Role: ACLRole(rule.Role), - } - } owner := "" if o.Owner != nil { owner = o.Owner.Entity @@ -895,27 +805,31 @@ func newObject(o *raw.Object) *ObjectAttrs { sha256 = o.CustomerEncryption.KeySha256 } return &ObjectAttrs{ - Bucket: o.Bucket, - Name: o.Name, - ContentType: o.ContentType, - ContentLanguage: o.ContentLanguage, - CacheControl: o.CacheControl, - ACL: acl, - Owner: owner, - ContentEncoding: o.ContentEncoding, - ContentDisposition: o.ContentDisposition, - Size: int64(o.Size), - MD5: md5, - CRC32C: crc32c, - MediaLink: o.MediaLink, - Metadata: o.Metadata, - Generation: o.Generation, - Metageneration: o.Metageneration, - StorageClass: o.StorageClass, - CustomerKeySHA256: sha256, - Created: convertTime(o.TimeCreated), - Deleted: convertTime(o.TimeDeleted), - Updated: convertTime(o.Updated), + Bucket: o.Bucket, + Name: o.Name, + ContentType: o.ContentType, + ContentLanguage: o.ContentLanguage, + CacheControl: o.CacheControl, + EventBasedHold: o.EventBasedHold, + TemporaryHold: o.TemporaryHold, + RetentionExpirationTime: convertTime(o.RetentionExpirationTime), + ACL: toObjectACLRules(o.Acl), + Owner: owner, + ContentEncoding: o.ContentEncoding, + ContentDisposition: o.ContentDisposition, + Size: int64(o.Size), + MD5: md5, + CRC32C: crc32c, + MediaLink: o.MediaLink, + Metadata: o.Metadata, + Generation: o.Generation, + Metageneration: o.Metageneration, + StorageClass: o.StorageClass, + CustomerKeySHA256: sha256, + KMSKeyName: o.KmsKeyName, + Created: convertTime(o.TimeCreated), + Deleted: convertTime(o.TimeDeleted), + Updated: convertTime(o.Updated), } } @@ -1198,4 +1112,12 @@ func setEncryptionHeaders(headers http.Header, key []byte, copySource bool) erro return nil } -// TODO(jbd): Add storage.objects.watch. +// ServiceAccount fetches the email address of the given project's Google Cloud Storage service account. +func (c *Client) ServiceAccount(ctx context.Context, projectID string) (string, error) { + r := c.raw.Projects.ServiceAccount.Get(projectID) + res, err := r.Context(ctx).Do() + if err != nil { + return "", err + } + return res.EmailAddress, nil +} diff --git a/vendor/cloud.google.com/go/storage/writer.go b/vendor/cloud.google.com/go/storage/writer.go index 2c6f9cb4a..3a58c404e 100644 --- a/vendor/cloud.google.com/go/storage/writer.go +++ b/vendor/cloud.google.com/go/storage/writer.go @@ -1,4 +1,4 @@ -// Copyright 2014 Google Inc. All Rights Reserved. +// Copyright 2014 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,6 +15,7 @@ package storage import ( + "context" "encoding/base64" "errors" "fmt" @@ -22,7 +23,6 @@ import ( "sync" "unicode/utf8" - "golang.org/x/net/context" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" ) @@ -48,8 +48,11 @@ type Writer struct { // to the nearest multiple of 256K. If zero, chunking will be disabled and // the object will be uploaded in a single request. // - // ChunkSize will default to a reasonable value. Any custom configuration - // must be done before the first Write call. + // ChunkSize will default to a reasonable value. If you perform many concurrent + // writes of small objects, you may wish set ChunkSize to a value that matches + // your objects' sizes to avoid consuming large amounts of memory. + // + // ChunkSize must be set before the first Write call. ChunkSize int // ProgressFunc can be used to monitor the progress of a large write. @@ -85,10 +88,15 @@ func (w *Writer) open() error { if !utf8.ValidString(attrs.Name) { return fmt.Errorf("storage: object name %q is not valid UTF-8", attrs.Name) } + if attrs.KMSKeyName != "" && w.o.encryptionKey != nil { + return errors.New("storage: cannot use KMSKeyName with a customer-supplied encryption key") + } pr, pw := io.Pipe() w.pw = pw w.opened = true + go w.monitorCancel() + if w.ChunkSize < 0 { return errors.New("storage: Writer.ChunkSize must be non-negative") } @@ -116,6 +124,12 @@ func (w *Writer) open() error { if w.ProgressFunc != nil { call.ProgressUpdater(func(n, _ int64) { w.ProgressFunc(n) }) } + if attrs.KMSKeyName != "" { + call.KmsKeyName(attrs.KMSKeyName) + } + if attrs.PredefinedACL != "" { + call.PredefinedAcl(attrs.PredefinedACL) + } if err := setEncryptionHeaders(call.Header(), w.o.encryptionKey, false); err != nil { w.mu.Lock() w.err = err @@ -176,7 +190,19 @@ func (w *Writer) Write(p []byte) (n int, err error) { return 0, err } } - return w.pw.Write(p) + n, err = w.pw.Write(p) + if err != nil { + w.mu.Lock() + werr := w.err + w.mu.Unlock() + // Preserve existing functionality that when context is canceled, Write will return + // context.Canceled instead of "io: read/write on closed pipe". This hides the + // pipe implementation detail from users and makes Write seem as though it's an RPC. + if werr == context.Canceled || werr == context.DeadlineExceeded { + return n, werr + } + } + return n, err } // Close completes the write operation and flushes any buffered data. @@ -188,15 +214,35 @@ func (w *Writer) Close() error { return err } } + + // Closing either the read or write causes the entire pipe to close. if err := w.pw.Close(); err != nil { return err } + <-w.donec w.mu.Lock() defer w.mu.Unlock() return w.err } +// monitorCancel is intended to be used as a background goroutine. It monitors the +// the context, and when it observes that the context has been canceled, it manually +// closes things that do not take a context. +func (w *Writer) monitorCancel() { + select { + case <-w.ctx.Done(): + w.mu.Lock() + werr := w.ctx.Err() + w.err = werr + w.mu.Unlock() + + // Closing either the read or write causes the entire pipe to close. + w.CloseWithError(werr) + case <-w.donec: + } +} + // CloseWithError aborts the write operation with the provided error. // CloseWithError always returns nil. // diff --git a/vendor/github.com/aws/aws-sdk-go/aws/awsutil/string_value.go b/vendor/github.com/aws/aws-sdk-go/aws/awsutil/string_value.go index b6432f1a1..645df2450 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/awsutil/string_value.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/awsutil/string_value.go @@ -23,28 +23,27 @@ func stringValue(v reflect.Value, indent int, buf *bytes.Buffer) { case reflect.Struct: buf.WriteString("{\n") - names := []string{} for i := 0; i < v.Type().NumField(); i++ { - name := v.Type().Field(i).Name - f := v.Field(i) - if name[0:1] == strings.ToLower(name[0:1]) { + ft := v.Type().Field(i) + fv := v.Field(i) + + if ft.Name[0:1] == strings.ToLower(ft.Name[0:1]) { continue // ignore unexported fields } - if (f.Kind() == reflect.Ptr || f.Kind() == reflect.Slice) && f.IsNil() { + if (fv.Kind() == reflect.Ptr || fv.Kind() == reflect.Slice) && fv.IsNil() { continue // ignore unset fields } - names = append(names, name) - } - for i, n := range names { - val := v.FieldByName(n) buf.WriteString(strings.Repeat(" ", indent+2)) - buf.WriteString(n + ": ") - stringValue(val, indent+2, buf) + buf.WriteString(ft.Name + ": ") - if i < len(names)-1 { - buf.WriteString(",\n") + if tag := ft.Tag.Get("sensitive"); tag == "true" { + buf.WriteString("") + } else { + stringValue(fv, indent+2, buf) } + + buf.WriteString(",\n") } buf.WriteString("\n" + strings.Repeat(" ", indent) + "}") diff --git a/vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go b/vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go new file mode 100644 index 000000000..f56e70c73 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go @@ -0,0 +1,425 @@ +/* +Package processcreds is a credential Provider to retrieve `credential_process` +credentials. + +WARNING: The following describes a method of sourcing credentials from an external +process. This can potentially be dangerous, so proceed with caution. Other +credential providers should be preferred if at all possible. If using this +option, you should make sure that the config file is as locked down as possible +using security best practices for your operating system. + +You can use credentials from a `credential_process` in a variety of ways. + +One way is to setup your shared config file, located in the default +location, with the `credential_process` key and the command you want to be +called. You also need to set the AWS_SDK_LOAD_CONFIG environment variable +(e.g., `export AWS_SDK_LOAD_CONFIG=1`) to use the shared config file. + + [default] + credential_process = /command/to/call + +Creating a new session will use the credential process to retrieve credentials. +NOTE: If there are credentials in the profile you are using, the credential +process will not be used. + + // Initialize a session to load credentials. + sess, _ := session.NewSession(&aws.Config{ + Region: aws.String("us-east-1")}, + ) + + // Create S3 service client to use the credentials. + svc := s3.New(sess) + +Another way to use the `credential_process` method is by using +`credentials.NewCredentials()` and providing a command to be executed to +retrieve credentials: + + // Create credentials using the ProcessProvider. + creds := processcreds.NewCredentials("/path/to/command") + + // Create service client value configured for credentials. + svc := s3.New(sess, &aws.Config{Credentials: creds}) + +You can set a non-default timeout for the `credential_process` with another +constructor, `credentials.NewCredentialsTimeout()`, providing the timeout. To +set a one minute timeout: + + // Create credentials using the ProcessProvider. + creds := processcreds.NewCredentialsTimeout( + "/path/to/command", + time.Duration(500) * time.Millisecond) + +If you need more control, you can set any configurable options in the +credentials using one or more option functions. For example, you can set a two +minute timeout, a credential duration of 60 minutes, and a maximum stdout +buffer size of 2k. + + creds := processcreds.NewCredentials( + "/path/to/command", + func(opt *ProcessProvider) { + opt.Timeout = time.Duration(2) * time.Minute + opt.Duration = time.Duration(60) * time.Minute + opt.MaxBufSize = 2048 + }) + +You can also use your own `exec.Cmd`: + + // Create an exec.Cmd + myCommand := exec.Command("/path/to/command") + + // Create credentials using your exec.Cmd and custom timeout + creds := processcreds.NewCredentialsCommand( + myCommand, + func(opt *processcreds.ProcessProvider) { + opt.Timeout = time.Duration(1) * time.Second + }) +*/ +package processcreds + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" +) + +const ( + // ProviderName is the name this credentials provider will label any + // returned credentials Value with. + ProviderName = `ProcessProvider` + + // ErrCodeProcessProviderParse error parsing process output + ErrCodeProcessProviderParse = "ProcessProviderParseError" + + // ErrCodeProcessProviderVersion version error in output + ErrCodeProcessProviderVersion = "ProcessProviderVersionError" + + // ErrCodeProcessProviderRequired required attribute missing in output + ErrCodeProcessProviderRequired = "ProcessProviderRequiredError" + + // ErrCodeProcessProviderExecution execution of command failed + ErrCodeProcessProviderExecution = "ProcessProviderExecutionError" + + // errMsgProcessProviderTimeout process took longer than allowed + errMsgProcessProviderTimeout = "credential process timed out" + + // errMsgProcessProviderProcess process error + errMsgProcessProviderProcess = "error in credential_process" + + // errMsgProcessProviderParse problem parsing output + errMsgProcessProviderParse = "parse failed of credential_process output" + + // errMsgProcessProviderVersion version error in output + errMsgProcessProviderVersion = "wrong version in process output (not 1)" + + // errMsgProcessProviderMissKey missing access key id in output + errMsgProcessProviderMissKey = "missing AccessKeyId in process output" + + // errMsgProcessProviderMissSecret missing secret acess key in output + errMsgProcessProviderMissSecret = "missing SecretAccessKey in process output" + + // errMsgProcessProviderPrepareCmd prepare of command failed + errMsgProcessProviderPrepareCmd = "failed to prepare command" + + // errMsgProcessProviderEmptyCmd command must not be empty + errMsgProcessProviderEmptyCmd = "command must not be empty" + + // errMsgProcessProviderPipe failed to initialize pipe + errMsgProcessProviderPipe = "failed to initialize pipe" + + // DefaultDuration is the default amount of time in minutes that the + // credentials will be valid for. + DefaultDuration = time.Duration(15) * time.Minute + + // DefaultBufSize limits buffer size from growing to an enormous + // amount due to a faulty process. + DefaultBufSize = 512 + + // DefaultTimeout default limit on time a process can run. + DefaultTimeout = time.Duration(1) * time.Minute +) + +// ProcessProvider satisfies the credentials.Provider interface, and is a +// client to retrieve credentials from a process. +type ProcessProvider struct { + staticCreds bool + credentials.Expiry + originalCommand []string + + // Expiry duration of the credentials. Defaults to 15 minutes if not set. + Duration time.Duration + + // ExpiryWindow will allow the credentials to trigger refreshing prior to + // the credentials actually expiring. This is beneficial so race conditions + // with expiring credentials do not cause request to fail unexpectedly + // due to ExpiredTokenException exceptions. + // + // So a ExpiryWindow of 10s would cause calls to IsExpired() to return true + // 10 seconds before the credentials are actually expired. + // + // If ExpiryWindow is 0 or less it will be ignored. + ExpiryWindow time.Duration + + // A string representing an os command that should return a JSON with + // credential information. + command *exec.Cmd + + // MaxBufSize limits memory usage from growing to an enormous + // amount due to a faulty process. + MaxBufSize int + + // Timeout limits the time a process can run. + Timeout time.Duration +} + +// NewCredentials returns a pointer to a new Credentials object wrapping the +// ProcessProvider. The credentials will expire every 15 minutes by default. +func NewCredentials(command string, options ...func(*ProcessProvider)) *credentials.Credentials { + p := &ProcessProvider{ + command: exec.Command(command), + Duration: DefaultDuration, + Timeout: DefaultTimeout, + MaxBufSize: DefaultBufSize, + } + + for _, option := range options { + option(p) + } + + return credentials.NewCredentials(p) +} + +// NewCredentialsTimeout returns a pointer to a new Credentials object with +// the specified command and timeout, and default duration and max buffer size. +func NewCredentialsTimeout(command string, timeout time.Duration) *credentials.Credentials { + p := NewCredentials(command, func(opt *ProcessProvider) { + opt.Timeout = timeout + }) + + return p +} + +// NewCredentialsCommand returns a pointer to a new Credentials object with +// the specified command, and default timeout, duration and max buffer size. +func NewCredentialsCommand(command *exec.Cmd, options ...func(*ProcessProvider)) *credentials.Credentials { + p := &ProcessProvider{ + command: command, + Duration: DefaultDuration, + Timeout: DefaultTimeout, + MaxBufSize: DefaultBufSize, + } + + for _, option := range options { + option(p) + } + + return credentials.NewCredentials(p) +} + +type credentialProcessResponse struct { + Version int + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string + SessionToken string + Expiration *time.Time +} + +// Retrieve executes the 'credential_process' and returns the credentials. +func (p *ProcessProvider) Retrieve() (credentials.Value, error) { + out, err := p.executeCredentialProcess() + if err != nil { + return credentials.Value{ProviderName: ProviderName}, err + } + + // Serialize and validate response + resp := &credentialProcessResponse{} + if err = json.Unmarshal(out, resp); err != nil { + return credentials.Value{ProviderName: ProviderName}, awserr.New( + ErrCodeProcessProviderParse, + fmt.Sprintf("%s: %s", errMsgProcessProviderParse, string(out)), + err) + } + + if resp.Version != 1 { + return credentials.Value{ProviderName: ProviderName}, awserr.New( + ErrCodeProcessProviderVersion, + errMsgProcessProviderVersion, + nil) + } + + if len(resp.AccessKeyID) == 0 { + return credentials.Value{ProviderName: ProviderName}, awserr.New( + ErrCodeProcessProviderRequired, + errMsgProcessProviderMissKey, + nil) + } + + if len(resp.SecretAccessKey) == 0 { + return credentials.Value{ProviderName: ProviderName}, awserr.New( + ErrCodeProcessProviderRequired, + errMsgProcessProviderMissSecret, + nil) + } + + // Handle expiration + p.staticCreds = resp.Expiration == nil + if resp.Expiration != nil { + p.SetExpiration(*resp.Expiration, p.ExpiryWindow) + } + + return credentials.Value{ + ProviderName: ProviderName, + AccessKeyID: resp.AccessKeyID, + SecretAccessKey: resp.SecretAccessKey, + SessionToken: resp.SessionToken, + }, nil +} + +// IsExpired returns true if the credentials retrieved are expired, or not yet +// retrieved. +func (p *ProcessProvider) IsExpired() bool { + if p.staticCreds { + return false + } + return p.Expiry.IsExpired() +} + +// prepareCommand prepares the command to be executed. +func (p *ProcessProvider) prepareCommand() error { + + var cmdArgs []string + if runtime.GOOS == "windows" { + cmdArgs = []string{"cmd.exe", "/C"} + } else { + cmdArgs = []string{"sh", "-c"} + } + + if len(p.originalCommand) == 0 { + p.originalCommand = make([]string, len(p.command.Args)) + copy(p.originalCommand, p.command.Args) + + // check for empty command because it succeeds + if len(strings.TrimSpace(p.originalCommand[0])) < 1 { + return awserr.New( + ErrCodeProcessProviderExecution, + fmt.Sprintf( + "%s: %s", + errMsgProcessProviderPrepareCmd, + errMsgProcessProviderEmptyCmd), + nil) + } + } + + cmdArgs = append(cmdArgs, p.originalCommand...) + p.command = exec.Command(cmdArgs[0], cmdArgs[1:]...) + p.command.Env = os.Environ() + + return nil +} + +// executeCredentialProcess starts the credential process on the OS and +// returns the results or an error. +func (p *ProcessProvider) executeCredentialProcess() ([]byte, error) { + + if err := p.prepareCommand(); err != nil { + return nil, err + } + + // Setup the pipes + outReadPipe, outWritePipe, err := os.Pipe() + if err != nil { + return nil, awserr.New( + ErrCodeProcessProviderExecution, + errMsgProcessProviderPipe, + err) + } + + p.command.Stderr = os.Stderr // display stderr on console for MFA + p.command.Stdout = outWritePipe // get creds json on process's stdout + p.command.Stdin = os.Stdin // enable stdin for MFA + + output := bytes.NewBuffer(make([]byte, 0, p.MaxBufSize)) + + stdoutCh := make(chan error, 1) + go readInput( + io.LimitReader(outReadPipe, int64(p.MaxBufSize)), + output, + stdoutCh) + + execCh := make(chan error, 1) + go executeCommand(*p.command, execCh) + + finished := false + var errors []error + for !finished { + select { + case readError := <-stdoutCh: + errors = appendError(errors, readError) + finished = true + case execError := <-execCh: + err := outWritePipe.Close() + errors = appendError(errors, err) + errors = appendError(errors, execError) + if errors != nil { + return output.Bytes(), awserr.NewBatchError( + ErrCodeProcessProviderExecution, + errMsgProcessProviderProcess, + errors) + } + case <-time.After(p.Timeout): + finished = true + return output.Bytes(), awserr.NewBatchError( + ErrCodeProcessProviderExecution, + errMsgProcessProviderTimeout, + errors) // errors can be nil + } + } + + out := output.Bytes() + + if runtime.GOOS == "windows" { + // windows adds slashes to quotes + out = []byte(strings.Replace(string(out), `\"`, `"`, -1)) + } + + return out, nil +} + +// appendError conveniently checks for nil before appending slice +func appendError(errors []error, err error) []error { + if err != nil { + return append(errors, err) + } + return errors +} + +func executeCommand(cmd exec.Cmd, exec chan error) { + // Start the command + err := cmd.Start() + if err == nil { + err = cmd.Wait() + } + + exec <- err +} + +func readInput(r io.Reader, w io.Writer, read chan error) { + tee := io.TeeReader(r, w) + + _, err := ioutil.ReadAll(tee) + + if err == io.EOF { + err = nil + } + + read <- err // will only arrive here when write end of pipe is closed +} diff --git a/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go b/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go index 118618442..9a5697c25 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go @@ -227,9 +227,7 @@ func (rep *Reporter) InjectHandlers(handlers *request.Handlers) { apiCallAttemptHandler := request.NamedHandler{Name: APICallAttemptMetricHandlerName, Fn: rep.sendAPICallAttemptMetric} handlers.Complete.PushFrontNamed(apiCallHandler) - handlers.Complete.PushFrontNamed(apiCallAttemptHandler) - - handlers.AfterRetry.PushFrontNamed(apiCallAttemptHandler) + handlers.CompleteAttempt.PushFrontNamed(apiCallAttemptHandler) } // boolIntValue return 1 for true and 0 for false. diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index c01c90185..a97b62946 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -170,6 +170,7 @@ const ( SupportServiceID = "support" // Support. SwfServiceID = "swf" // Swf. TaggingServiceID = "tagging" // Tagging. + TransferServiceID = "transfer" // Transfer. TranslateServiceID = "translate" // Translate. WafServiceID = "waf" // Waf. WafRegionalServiceID = "waf-regional" // WafRegional. @@ -1868,6 +1869,7 @@ var awsPartition = partition{ Endpoints: endpoints{ "eu-west-1": endpoint{}, "us-east-1": endpoint{}, + "us-west-2": endpoint{}, }, }, "polly": service{ @@ -2702,6 +2704,24 @@ var awsPartition = partition{ "us-west-2": endpoint{}, }, }, + "transfer": service{ + + Endpoints: endpoints{ + "ap-northeast-1": endpoint{}, + "ap-northeast-2": endpoint{}, + "ap-southeast-1": endpoint{}, + "ap-southeast-2": endpoint{}, + "ca-central-1": endpoint{}, + "eu-central-1": endpoint{}, + "eu-west-1": endpoint{}, + "eu-west-2": endpoint{}, + "eu-west-3": endpoint{}, + "us-east-1": endpoint{}, + "us-east-2": endpoint{}, + "us-west-1": endpoint{}, + "us-west-2": endpoint{}, + }, + }, "translate": service{ Defaults: endpoint{ Protocols: []string{"https"}, @@ -3088,6 +3108,12 @@ var awscnPartition = partition{ "cn-northwest-1": endpoint{}, }, }, + "polly": service{ + + Endpoints: endpoints{ + "cn-northwest-1": endpoint{}, + }, + }, "rds": service{ Endpoints: endpoints{ @@ -3320,6 +3346,12 @@ var awsusgovPartition = partition{ Endpoints: endpoints{ "us-gov-east-1": endpoint{}, + "us-gov-east-1-fips": endpoint{ + Hostname: "codedeploy-fips.us-gov-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-east-1", + }, + }, "us-gov-west-1": endpoint{}, "us-gov-west-1-fips": endpoint{ Hostname: "codedeploy-fips.us-gov-west-1.amazonaws.com", diff --git a/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go b/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go index 605a72d3c..8ef8548a9 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go @@ -19,6 +19,7 @@ type Handlers struct { UnmarshalError HandlerList Retry HandlerList AfterRetry HandlerList + CompleteAttempt HandlerList Complete HandlerList } @@ -36,6 +37,7 @@ func (h *Handlers) Copy() Handlers { UnmarshalMeta: h.UnmarshalMeta.copy(), Retry: h.Retry.copy(), AfterRetry: h.AfterRetry.copy(), + CompleteAttempt: h.CompleteAttempt.copy(), Complete: h.Complete.copy(), } } @@ -53,6 +55,7 @@ func (h *Handlers) Clear() { h.ValidateResponse.Clear() h.Retry.Clear() h.AfterRetry.Clear() + h.CompleteAttempt.Clear() h.Complete.Clear() } diff --git a/vendor/github.com/aws/aws-sdk-go/aws/request/request.go b/vendor/github.com/aws/aws-sdk-go/aws/request/request.go index 63e7f71c3..f3ebce05b 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/request/request.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/request/request.go @@ -266,9 +266,9 @@ func (r *Request) SetReaderBody(reader io.ReadSeeker) { } // Presign returns the request's signed URL. Error will be returned -// if the signing fails. The expire parameter is only used for presigned Amazon +// if the signing fails. The expire parameter is only used for presigned Amazon // S3 API requests. All other AWS services will use a fixed expriation -// time of 15 minutes. +// time of 15 minutes. // // It is invalid to create a presigned URL with a expire duration 0 or less. An // error is returned if expire duration is 0 or less. @@ -466,80 +466,78 @@ func (r *Request) Send() error { r.Handlers.Complete.Run(r) }() + if err := r.Error; err != nil { + return err + } + for { + r.Error = nil r.AttemptTime = time.Now() - if aws.BoolValue(r.Retryable) { - if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { - r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", - r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) - } - - // The previous http.Request will have a reference to the r.Body - // and the HTTP Client's Transport may still be reading from - // the request's body even though the Client's Do returned. - r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) - r.ResetBody() - - // Closing response body to ensure that no response body is leaked - // between retry attempts. - if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { - r.HTTPResponse.Body.Close() - } - } - r.Sign() - if r.Error != nil { - return r.Error + if err := r.Sign(); err != nil { + debugLogReqError(r, "Sign Request", false, err) + return err } - r.Retryable = nil - - r.Handlers.Send.Run(r) - if r.Error != nil { - if !shouldRetryCancel(r) { - return r.Error - } - - err := r.Error + if err := r.sendRequest(); err == nil { + return nil + } else if !shouldRetryCancel(r) { + return err + } else { r.Handlers.Retry.Run(r) r.Handlers.AfterRetry.Run(r) - if r.Error != nil { - debugLogReqError(r, "Send Request", false, err) - return r.Error - } - debugLogReqError(r, "Send Request", true, err) - continue - } - r.Handlers.UnmarshalMeta.Run(r) - r.Handlers.ValidateResponse.Run(r) - if r.Error != nil { - r.Handlers.UnmarshalError.Run(r) - err := r.Error - r.Handlers.Retry.Run(r) - r.Handlers.AfterRetry.Run(r) - if r.Error != nil { - debugLogReqError(r, "Validate Response", false, err) + if r.Error != nil || !aws.BoolValue(r.Retryable) { return r.Error } - debugLogReqError(r, "Validate Response", true, err) - continue - } - r.Handlers.Unmarshal.Run(r) - if r.Error != nil { - err := r.Error - r.Handlers.Retry.Run(r) - r.Handlers.AfterRetry.Run(r) - if r.Error != nil { - debugLogReqError(r, "Unmarshal Response", false, err) - return r.Error - } - debugLogReqError(r, "Unmarshal Response", true, err) + r.prepareRetry() continue } + } +} + +func (r *Request) prepareRetry() { + if r.Config.LogLevel.Matches(aws.LogDebugWithRequestRetries) { + r.Config.Logger.Log(fmt.Sprintf("DEBUG: Retrying Request %s/%s, attempt %d", + r.ClientInfo.ServiceName, r.Operation.Name, r.RetryCount)) + } + + // The previous http.Request will have a reference to the r.Body + // and the HTTP Client's Transport may still be reading from + // the request's body even though the Client's Do returned. + r.HTTPRequest = copyHTTPRequest(r.HTTPRequest, nil) + r.ResetBody() + + // Closing response body to ensure that no response body is leaked + // between retry attempts. + if r.HTTPResponse != nil && r.HTTPResponse.Body != nil { + r.HTTPResponse.Body.Close() + } +} + +func (r *Request) sendRequest() (sendErr error) { + defer r.Handlers.CompleteAttempt.Run(r) + + r.Retryable = nil + r.Handlers.Send.Run(r) + if r.Error != nil { + debugLogReqError(r, "Send Request", r.WillRetry(), r.Error) + return r.Error + } - break + r.Handlers.UnmarshalMeta.Run(r) + r.Handlers.ValidateResponse.Run(r) + if r.Error != nil { + r.Handlers.UnmarshalError.Run(r) + debugLogReqError(r, "Validate Response", r.WillRetry(), r.Error) + return r.Error + } + + r.Handlers.Unmarshal.Run(r) + if r.Error != nil { + debugLogReqError(r, "Unmarshal Response", r.WillRetry(), r.Error) + return r.Error } return nil diff --git a/vendor/github.com/aws/aws-sdk-go/aws/session/doc.go b/vendor/github.com/aws/aws-sdk-go/aws/session/doc.go index 98d420fd6..2a0e882d0 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/session/doc.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/session/doc.go @@ -99,7 +99,7 @@ handler logs every request and its payload made by a service client: sess.Handlers.Send.PushFront(func(r *request.Request) { // Log every request made and its payload - logger.Println("Request: %s/%s, Payload: %s", + logger.Printf("Request: %s/%s, Payload: %s", r.ClientInfo.ServiceName, r.Operation, r.Params) }) diff --git a/vendor/github.com/aws/aws-sdk-go/aws/session/session.go b/vendor/github.com/aws/aws-sdk-go/aws/session/session.go index e7c156e8b..9bdbafd65 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/session/session.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/session/session.go @@ -14,6 +14,7 @@ import ( "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/corehandlers" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/processcreds" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/csm" "github.com/aws/aws-sdk-go/aws/defaults" @@ -534,6 +535,10 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config, envCfg envConfig, sharedCfg share cfg.Credentials = credentials.NewStaticCredentialsFromCreds( sharedCfg.Creds, ) + } else if len(sharedCfg.CredentialProcess) > 0 { + cfg.Credentials = processcreds.NewCredentials( + sharedCfg.CredentialProcess, + ) } else { // Fallback to default credentials provider, include mock errors // for the credential chain so user can identify why credentials diff --git a/vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go b/vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go index 427b8a4e9..7cb44021b 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go @@ -28,6 +28,8 @@ const ( // endpoint discovery group enableEndpointDiscoveryKey = `endpoint_discovery_enabled` // optional + // External Credential Process + credentialProcessKey = `credential_process` // DefaultSharedConfigProfile is the default profile to be used when // loading configuration from the config files if another profile name @@ -60,6 +62,9 @@ type sharedConfig struct { AssumeRole assumeRoleConfig AssumeRoleSource *sharedConfig + // An external process to request credentials + CredentialProcess string + // Region is the region the SDK should use for looking up AWS service endpoints // and signing requests. // @@ -223,6 +228,11 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile) e } } + // `credential_process` + if credProc := section.String(credentialProcessKey); len(credProc) > 0 { + cfg.CredentialProcess = credProc + } + // Region if v := section.String(regionKey); len(v) > 0 { cfg.Region = v diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index ea1123147..6f7a3a73a 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.15.87" +const SDKVersion = "1.16.1" diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/api.go b/vendor/github.com/aws/aws-sdk-go/service/s3/api.go index 647191990..5c33ad0c4 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/s3/api.go @@ -7103,6 +7103,9 @@ func (s *AbortMultipartUploadInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -8105,6 +8108,9 @@ func (s *CompleteMultipartUploadInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -8180,7 +8186,7 @@ type CompleteMultipartUploadOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -8435,7 +8441,7 @@ type CopyObjectInput struct { // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt // the source object. The encryption key provided in this header must be one // that was used when the source object was created. - CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string"` + CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -8490,7 +8496,7 @@ type CopyObjectInput struct { // does not store the encryption key. The key must be appropriate for use with // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -8501,7 +8507,7 @@ type CopyObjectInput struct { // requests for an object protected by AWS KMS will fail if not made via SSL // or using SigV4. Documentation on configuring any of the officially supported // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -8541,6 +8547,9 @@ func (s *CopyObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.CopySource == nil { invalidParams.Add(request.NewErrParamRequired("CopySource")) } @@ -8820,7 +8829,7 @@ type CopyObjectOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -9027,6 +9036,9 @@ func (s *CreateBucketInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -9187,7 +9199,7 @@ type CreateMultipartUploadInput struct { // does not store the encryption key. The key must be appropriate for use with // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -9198,7 +9210,7 @@ type CreateMultipartUploadInput struct { // requests for an object protected by AWS KMS will fail if not made via SSL // or using SigV4. Documentation on configuring any of the officially supported // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -9232,6 +9244,9 @@ func (s *CreateMultipartUploadInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -9447,7 +9462,7 @@ type CreateMultipartUploadOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -9664,6 +9679,9 @@ func (s *DeleteBucketAnalyticsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -9730,6 +9748,9 @@ func (s *DeleteBucketCorsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -9790,6 +9811,9 @@ func (s *DeleteBucketEncryptionInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -9847,6 +9871,9 @@ func (s *DeleteBucketInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -9897,6 +9924,9 @@ func (s *DeleteBucketInventoryConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -9963,6 +9993,9 @@ func (s *DeleteBucketLifecycleInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10027,6 +10060,9 @@ func (s *DeleteBucketMetricsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -10107,6 +10143,9 @@ func (s *DeleteBucketPolicyInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10169,6 +10208,9 @@ func (s *DeleteBucketReplicationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10226,6 +10268,9 @@ func (s *DeleteBucketTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10283,6 +10328,9 @@ func (s *DeleteBucketWebsiteInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10409,6 +10457,8 @@ type DeleteObjectInput struct { // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Indicates whether S3 Object Lock should bypass Governance-mode restrictions + // to process this operation. BypassGovernanceRetention *bool `location:"header" locationName:"x-amz-bypass-governance-retention" type:"boolean"` // Key is a required field @@ -10444,6 +10494,9 @@ func (s *DeleteObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -10573,6 +10626,9 @@ func (s *DeleteObjectTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -10675,6 +10731,9 @@ func (s *DeleteObjectsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Delete == nil { invalidParams.Add(request.NewErrParamRequired("Delete")) } @@ -10792,6 +10851,9 @@ func (s *DeletePublicAccessBlockInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -10994,7 +11056,7 @@ type Encryption struct { // If the encryption type is aws:kms, this optional value specifies the AWS // KMS key ID to use for encryption of job results. - KMSKeyId *string `type:"string"` + KMSKeyId *string `type:"string" sensitive:"true"` } // String returns the string representation @@ -11239,6 +11301,9 @@ func (s *GetBucketAccelerateConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11305,6 +11370,9 @@ func (s *GetBucketAclInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11386,6 +11454,9 @@ func (s *GetBucketAnalyticsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -11461,6 +11532,9 @@ func (s *GetBucketCorsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11529,6 +11603,9 @@ func (s *GetBucketEncryptionInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11603,6 +11680,9 @@ func (s *GetBucketInventoryConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -11678,6 +11758,9 @@ func (s *GetBucketLifecycleConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11743,6 +11826,9 @@ func (s *GetBucketLifecycleInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11808,6 +11894,9 @@ func (s *GetBucketLocationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11873,6 +11962,9 @@ func (s *GetBucketLoggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -11948,6 +12040,9 @@ func (s *GetBucketMetricsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -12025,6 +12120,9 @@ func (s *GetBucketNotificationConfigurationRequest) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12068,6 +12166,9 @@ func (s *GetBucketPolicyInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12136,6 +12237,9 @@ func (s *GetBucketPolicyStatusInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12202,6 +12306,9 @@ func (s *GetBucketReplicationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12269,6 +12376,9 @@ func (s *GetBucketRequestPaymentInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12335,6 +12445,9 @@ func (s *GetBucketTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12401,6 +12514,9 @@ func (s *GetBucketVersioningInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12478,6 +12594,9 @@ func (s *GetBucketWebsiteInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -12579,6 +12698,9 @@ func (s *GetObjectAclInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -12730,7 +12852,7 @@ type GetObjectInput struct { // does not store the encryption key. The key must be appropriate for use with // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -12757,6 +12879,9 @@ func (s *GetObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -12937,6 +13062,9 @@ func (s *GetObjectLegalHoldInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -13029,6 +13157,9 @@ func (s *GetObjectLockConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -13132,6 +13263,8 @@ type GetObjectOutput struct { // you can create metadata whose values are not legal HTTP headers. MissingMeta *int64 `location:"header" locationName:"x-amz-missing-meta" type:"integer"` + // Indicates whether this object has an active legal hold. This field is only + // returned if you have permission to view an object's legal hold status. ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"` // The Object Lock mode currently in place for this object. @@ -13165,7 +13298,7 @@ type GetObjectOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -13420,6 +13553,9 @@ func (s *GetObjectRetentionInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -13515,6 +13651,9 @@ func (s *GetObjectTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -13616,6 +13755,9 @@ func (s *GetObjectTorrentInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -13712,6 +13854,9 @@ func (s *GetPublicAccessBlockInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -13937,6 +14082,9 @@ func (s *HeadBucketInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -14020,7 +14168,7 @@ type HeadObjectInput struct { // does not store the encryption key. The key must be appropriate for use with // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -14047,6 +14195,9 @@ func (s *HeadObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -14240,7 +14391,7 @@ type HeadObjectOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -15458,6 +15609,9 @@ func (s *ListBucketAnalyticsConfigurationsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -15569,6 +15723,9 @@ func (s *ListBucketInventoryConfigurationsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -15680,6 +15837,9 @@ func (s *ListBucketMetricsConfigurationsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -15858,6 +16018,9 @@ func (s *ListMultipartUploadsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -16094,6 +16257,9 @@ func (s *ListObjectVersionsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -16326,6 +16492,9 @@ func (s *ListObjectsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -16543,6 +16712,9 @@ func (s *ListObjectsV2Input) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -16788,6 +16960,9 @@ func (s *ListPartsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -17805,7 +17980,7 @@ func (s *ObjectIdentifier) SetVersionId(v string) *ObjectIdentifier { type ObjectLockConfiguration struct { _ struct{} `type:"structure"` - // Indicates whether this object has an Object Lock configuration enabled. + // Indicates whether this bucket has an Object Lock configuration enabled. ObjectLockEnabled *string `type:"string" enum:"ObjectLockEnabled"` // The Object Lock rule in place for the specified object. @@ -17865,6 +18040,7 @@ type ObjectLockRetention struct { // Indicates the Retention mode for the specified object. Mode *string `type:"string" enum:"ObjectLockRetentionMode"` + // The date on which this Object Lock Retention will expire. RetainUntilDate *time.Time `type:"timestamp" timestampFormat:"iso8601"` } @@ -18381,6 +18557,9 @@ func (s *PutBucketAccelerateConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -18465,6 +18644,9 @@ func (s *PutBucketAclInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.AccessControlPolicy != nil { if err := s.AccessControlPolicy.Validate(); err != nil { invalidParams.AddNested("AccessControlPolicy", err.(request.ErrInvalidParams)) @@ -18584,6 +18766,9 @@ func (s *PutBucketAnalyticsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -18664,6 +18849,9 @@ func (s *PutBucketCorsInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.CORSConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("CORSConfiguration")) } @@ -18744,6 +18932,9 @@ func (s *PutBucketEncryptionInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.ServerSideEncryptionConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("ServerSideEncryptionConfiguration")) } @@ -18827,6 +19018,9 @@ func (s *PutBucketInventoryConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -18909,6 +19103,9 @@ func (s *PutBucketLifecycleConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.LifecycleConfiguration != nil { if err := s.LifecycleConfiguration.Validate(); err != nil { invalidParams.AddNested("LifecycleConfiguration", err.(request.ErrInvalidParams)) @@ -18979,6 +19176,9 @@ func (s *PutBucketLifecycleInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.LifecycleConfiguration != nil { if err := s.LifecycleConfiguration.Validate(); err != nil { invalidParams.AddNested("LifecycleConfiguration", err.(request.ErrInvalidParams)) @@ -19050,6 +19250,9 @@ func (s *PutBucketLoggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.BucketLoggingStatus == nil { invalidParams.Add(request.NewErrParamRequired("BucketLoggingStatus")) } @@ -19133,6 +19336,9 @@ func (s *PutBucketMetricsConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Id == nil { invalidParams.Add(request.NewErrParamRequired("Id")) } @@ -19219,6 +19425,9 @@ func (s *PutBucketNotificationConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.NotificationConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("NotificationConfiguration")) } @@ -19293,6 +19502,9 @@ func (s *PutBucketNotificationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.NotificationConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("NotificationConfiguration")) } @@ -19368,6 +19580,9 @@ func (s *PutBucketPolicyInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Policy == nil { invalidParams.Add(request.NewErrParamRequired("Policy")) } @@ -19446,6 +19661,9 @@ func (s *PutBucketReplicationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.ReplicationConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("ReplicationConfiguration")) } @@ -19520,6 +19738,9 @@ func (s *PutBucketRequestPaymentInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.RequestPaymentConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("RequestPaymentConfiguration")) } @@ -19594,6 +19815,9 @@ func (s *PutBucketTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Tagging == nil { invalidParams.Add(request.NewErrParamRequired("Tagging")) } @@ -19672,6 +19896,9 @@ func (s *PutBucketVersioningInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.VersioningConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("VersioningConfiguration")) } @@ -19747,6 +19974,9 @@ func (s *PutBucketWebsiteInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.WebsiteConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("WebsiteConfiguration")) } @@ -19851,6 +20081,9 @@ func (s *PutObjectAclInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -20050,7 +20283,7 @@ type PutObjectInput struct { // does not store the encryption key. The key must be appropriate for use with // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -20061,7 +20294,7 @@ type PutObjectInput struct { // requests for an object protected by AWS KMS will fail if not made via SSL // or using SigV4. Documentation on configuring any of the officially supported // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -20096,6 +20329,9 @@ func (s *PutObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -20340,6 +20576,9 @@ func (s *PutObjectLegalHoldInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -20450,6 +20689,9 @@ func (s *PutObjectLockConfigurationInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if invalidParams.Len() > 0 { return invalidParams @@ -20538,7 +20780,7 @@ type PutObjectOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -20615,6 +20857,7 @@ type PutObjectRetentionInput struct { // Bucket is a required field Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` + // Indicates whether this operation should bypass Governance-mode restrictions.j BypassGovernanceRetention *bool `location:"header" locationName:"x-amz-bypass-governance-retention" type:"boolean"` // The key name for the object that you want to apply this Object Retention @@ -20653,6 +20896,9 @@ func (s *PutObjectRetentionInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -20764,6 +21010,9 @@ func (s *PutObjectTaggingInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -20873,6 +21122,9 @@ func (s *PutPublicAccessBlockInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.PublicAccessBlockConfiguration == nil { invalidParams.Add(request.NewErrParamRequired("PublicAccessBlockConfiguration")) } @@ -21631,6 +21883,9 @@ func (s *RestoreObjectInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -22000,7 +22255,7 @@ type SSEKMS struct { // key to use for encrypting Inventory reports. // // KeyId is a required field - KeyId *string `type:"string" required:"true"` + KeyId *string `type:"string" required:"true" sensitive:"true"` } // String returns the string representation @@ -22316,7 +22571,7 @@ type SelectObjectContentInput struct { // The SSE Customer Key. For more information, see Server-Side Encryption (Using // Customer-Provided Encryption Keys (http://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // The SSE Customer Key MD5. For more information, see Server-Side Encryption // (Using Customer-Provided Encryption Keys (http://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html). @@ -22339,6 +22594,9 @@ func (s *SelectObjectContentInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Expression == nil { invalidParams.Add(request.NewErrParamRequired("Expression")) } @@ -22569,7 +22827,7 @@ type ServerSideEncryptionByDefault struct { // KMS master key ID to use for the default encryption. This parameter is allowed // if SSEAlgorithm is aws:kms. - KMSMasterKeyID *string `type:"string"` + KMSMasterKeyID *string `type:"string" sensitive:"true"` // Server-side encryption algorithm to use for the default encryption. // @@ -23320,7 +23578,7 @@ type UploadPartCopyInput struct { // Specifies the customer-provided encryption key for Amazon S3 to use to decrypt // the source object. The encryption key provided in this header must be one // that was used when the source object was created. - CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string"` + CopySourceSSECustomerKey *string `location:"header" locationName:"x-amz-copy-source-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -23351,7 +23609,7 @@ type UploadPartCopyInput struct { // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. This must be the same encryption key specified in the initiate multipart // upload request. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -23380,6 +23638,9 @@ func (s *UploadPartCopyInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.CopySource == nil { invalidParams.Add(request.NewErrParamRequired("CopySource")) } @@ -23550,7 +23811,7 @@ type UploadPartCopyOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -23653,7 +23914,7 @@ type UploadPartInput struct { // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm // header. This must be the same encryption key specified in the initiate multipart // upload request. - SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string"` + SSECustomerKey *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. // Amazon S3 uses this header for a message integrity check to ensure the encryption @@ -23682,6 +23943,9 @@ func (s *UploadPartInput) Validate() error { if s.Bucket == nil { invalidParams.Add(request.NewErrParamRequired("Bucket")) } + if s.Bucket != nil && len(*s.Bucket) < 1 { + invalidParams.Add(request.NewErrParamMinLen("Bucket", 1)) + } if s.Key == nil { invalidParams.Add(request.NewErrParamRequired("Key")) } @@ -23803,7 +24067,7 @@ type UploadPartOutput struct { // If present, specifies the ID of the AWS Key Management Service (KMS) master // encryption key that was used for the object. - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string"` + SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` // The Server-side encryption algorithm used when storing this object in S3 // (e.g., AES256, aws:kms). @@ -24156,6 +24420,9 @@ const ( // InventoryFormatOrc is a InventoryFormat enum value InventoryFormatOrc = "ORC" + + // InventoryFormatParquet is a InventoryFormat enum value + InventoryFormatParquet = "Parquet" ) const ( diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/customizations.go b/vendor/github.com/aws/aws-sdk-go/service/s3/customizations.go index 6f560a409..95f245636 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/customizations.go +++ b/vendor/github.com/aws/aws-sdk-go/service/s3/customizations.go @@ -33,6 +33,7 @@ func defaultInitRequestFn(r *request.Request) { switch r.Operation.Name { case opPutBucketCors, opPutBucketLifecycle, opPutBucketPolicy, opPutBucketTagging, opDeleteObjects, opPutBucketLifecycleConfiguration, + opPutObjectLegalHold, opPutObjectRetention, opPutObjectLockConfiguration, opPutBucketReplication: // These S3 operations require Content-MD5 to be set r.Handlers.Build.PushBack(contentMD5) diff --git a/vendor/github.com/googleapis/google-cloud-go-testing/LICENSE b/vendor/github.com/googleapis/google-cloud-go-testing/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/googleapis/google-cloud-go-testing/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/adapters.go b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/adapters.go new file mode 100644 index 000000000..905e89ded --- /dev/null +++ b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/adapters.go @@ -0,0 +1,176 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stiface + +import ( + "context" + + "cloud.google.com/go/storage" +) + +// AdaptClient adapts a storage.Client so that it satisfies the Client +// interface. +func AdaptClient(c *storage.Client) Client { + return client{c} +} + +type ( + client struct{ *storage.Client } + bucketHandle struct{ *storage.BucketHandle } + objectHandle struct{ *storage.ObjectHandle } + bucketIterator struct{ *storage.BucketIterator } + objectIterator struct{ *storage.ObjectIterator } + reader struct{ *storage.Reader } + writer struct{ *storage.Writer } + copier struct{ *storage.Copier } + composer struct{ *storage.Composer } + aclHandle struct{ *storage.ACLHandle } +) + +func (client) embedToIncludeNewMethods() {} +func (bucketHandle) embedToIncludeNewMethods() {} +func (objectHandle) embedToIncludeNewMethods() {} +func (bucketIterator) embedToIncludeNewMethods() {} +func (objectIterator) embedToIncludeNewMethods() {} +func (writer) embedToIncludeNewMethods() {} +func (reader) embedToIncludeNewMethods() {} +func (copier) embedToIncludeNewMethods() {} +func (composer) embedToIncludeNewMethods() {} +func (aclHandle) embedToIncludeNewMethods() {} + +func (c client) Bucket(name string) BucketHandle { + return bucketHandle{c.Client.Bucket(name)} +} + +func (c client) Buckets(ctx context.Context, projectID string) BucketIterator { + return bucketIterator{c.Client.Buckets(ctx, projectID)} +} + +func (b bucketHandle) Object(name string) ObjectHandle { + return objectHandle{b.BucketHandle.Object(name)} +} + +func (b bucketHandle) If(conds storage.BucketConditions) BucketHandle { + return bucketHandle{b.BucketHandle.If(conds)} +} + +func (b bucketHandle) Objects(ctx context.Context, q *storage.Query) ObjectIterator { + return objectIterator{b.BucketHandle.Objects(ctx, q)} +} + +func (b bucketHandle) DefaultObjectACL() ACLHandle { + return aclHandle{b.BucketHandle.DefaultObjectACL()} +} + +func (b bucketHandle) ACL() ACLHandle { + return aclHandle{b.BucketHandle.ACL()} +} + +func (b bucketHandle) UserProject(projectID string) BucketHandle { + return bucketHandle{b.BucketHandle.UserProject(projectID)} +} + +func (bi bucketIterator) SetPrefix(s string) { + bi.BucketIterator.Prefix = s +} + +func (o objectHandle) ACL() ACLHandle { + return aclHandle{o.ObjectHandle.ACL()} +} + +func (o objectHandle) Generation(gen int64) ObjectHandle { + return objectHandle{o.ObjectHandle.Generation(gen)} +} + +func (o objectHandle) If(conds storage.Conditions) ObjectHandle { + return objectHandle{o.ObjectHandle.If(conds)} +} + +func (o objectHandle) Key(encryptionKey []byte) ObjectHandle { + return objectHandle{o.ObjectHandle.Key(encryptionKey)} +} + +func (o objectHandle) ReadCompressed(compressed bool) ObjectHandle { + return objectHandle{o.ObjectHandle.ReadCompressed(compressed)} +} + +func (o objectHandle) NewReader(ctx context.Context) (Reader, error) { + r, err := o.ObjectHandle.NewReader(ctx) + if err != nil { + return nil, err + } + return reader{r}, nil +} + +func (o objectHandle) NewRangeReader(ctx context.Context, offset, length int64) (Reader, error) { + r, err := o.ObjectHandle.NewRangeReader(ctx, offset, length) + if err != nil { + return nil, err + } + return reader{r}, nil +} + +func (o objectHandle) NewWriter(ctx context.Context) Writer { + return writer{o.ObjectHandle.NewWriter(ctx)} +} + +func (o objectHandle) CopierFrom(src ObjectHandle) Copier { + return copier{o.ObjectHandle.CopierFrom(src.(objectHandle).ObjectHandle)} +} + +func (o objectHandle) ComposerFrom(srcs ...ObjectHandle) Composer { + objs := make([]*storage.ObjectHandle, len(srcs)) + for i, s := range srcs { + objs[i] = s.(objectHandle).ObjectHandle + } + return composer{o.ObjectHandle.ComposerFrom(objs...)} +} + +func (w writer) ObjectAttrs() *storage.ObjectAttrs { + return &w.Writer.ObjectAttrs +} + +func (w writer) SetChunkSize(s int) { + w.ChunkSize = s +} + +func (w writer) SetProgressFunc(f func(int64)) { + w.ProgressFunc = f +} + +func (w writer) SetCRC32C(c uint32) { + w.CRC32C = c + w.SendCRC32C = true +} + +func (c copier) ObjectAttrs() *storage.ObjectAttrs { + return &c.Copier.ObjectAttrs +} + +func (c copier) SetRewriteToken(t string) { + c.RewriteToken = t +} + +func (c copier) SetProgressFunc(f func(copiedBytes, totalBytes uint64)) { + c.ProgressFunc = f +} + +func (c copier) SetDestinationKMSKeyName(k string) { + c.DestinationKMSKeyName = k +} + +func (c composer) ObjectAttrs() *storage.ObjectAttrs { + return &c.Composer.ObjectAttrs +} diff --git a/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/doc.go b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/doc.go new file mode 100644 index 000000000..3a777ea39 --- /dev/null +++ b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/doc.go @@ -0,0 +1,34 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package stiface provides a set of interfaces for the types in +// cloud.google.com/go/storage. These can be used to create mocks or other test +// doubles. The package also provides adapters to enable the types of the +// storage package to implement these interfaces. +// +// We do not recommend using mocks for most testing. Please read +// https://testing.googleblog.com/2013/05/testing-on-toilet-dont-overuse-mocks.html. +// +// Note: This package is in alpha. Some backwards-incompatible changes may occur. +// +// You must embed these interfaces to implement them: +// +// type ClientMock struct { +// stiface.Client +// ... +// } +// +// This ensures that your implementations will not break when methods are added +// to the interfaces. +package stiface diff --git a/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/interfaces.go b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/interfaces.go new file mode 100644 index 000000000..513ec4d0e --- /dev/null +++ b/vendor/github.com/googleapis/google-cloud-go-testing/storage/stiface/interfaces.go @@ -0,0 +1,133 @@ +// Copyright 2018 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stiface + +import ( + "context" + "io" + + "cloud.google.com/go/iam" + "cloud.google.com/go/storage" + "google.golang.org/api/iterator" +) + +type Client interface { + Bucket(name string) BucketHandle + Buckets(ctx context.Context, projectID string) BucketIterator + Close() error + + embedToIncludeNewMethods() +} + +type ObjectHandle interface { + ACL() ACLHandle + Generation(int64) ObjectHandle + If(storage.Conditions) ObjectHandle + Key([]byte) ObjectHandle + ReadCompressed(bool) ObjectHandle + Attrs(context.Context) (*storage.ObjectAttrs, error) + Update(context.Context, storage.ObjectAttrsToUpdate) (*storage.ObjectAttrs, error) + NewReader(context.Context) (Reader, error) + NewRangeReader(context.Context, int64, int64) (Reader, error) + NewWriter(context.Context) Writer + Delete(context.Context) error + CopierFrom(ObjectHandle) Copier + ComposerFrom(...ObjectHandle) Composer + + embedToIncludeNewMethods() +} + +type BucketHandle interface { + Create(context.Context, string, *storage.BucketAttrs) error + Delete(context.Context) error + DefaultObjectACL() ACLHandle + Object(string) ObjectHandle + Attrs(context.Context) (*storage.BucketAttrs, error) + Update(context.Context, storage.BucketAttrsToUpdate) (*storage.BucketAttrs, error) + If(storage.BucketConditions) BucketHandle + Objects(context.Context, *storage.Query) ObjectIterator + ACL() ACLHandle + IAM() *iam.Handle + UserProject(projectID string) BucketHandle + Notifications(context.Context) (map[string]*storage.Notification, error) + AddNotification(context.Context, *storage.Notification) (*storage.Notification, error) + DeleteNotification(context.Context, string) error + LockRetentionPolicy(context.Context) error + + embedToIncludeNewMethods() +} + +type ObjectIterator interface { + Next() (*storage.ObjectAttrs, error) + PageInfo() *iterator.PageInfo + + embedToIncludeNewMethods() +} + +type BucketIterator interface { + SetPrefix(string) + Next() (*storage.BucketAttrs, error) + PageInfo() *iterator.PageInfo + + embedToIncludeNewMethods() +} + +type ACLHandle interface { + Delete(context.Context, storage.ACLEntity) error + Set(context.Context, storage.ACLEntity, storage.ACLRole) error + List(context.Context) ([]storage.ACLRule, error) + + embedToIncludeNewMethods() +} + +type Reader interface { + io.ReadCloser + Size() int64 + Remain() int64 + ContentType() string + ContentEncoding() string + CacheControl() string + + embedToIncludeNewMethods() +} + +type Writer interface { + io.WriteCloser + ObjectAttrs() *storage.ObjectAttrs + SetChunkSize(int) + SetProgressFunc(func(int64)) + SetCRC32C(uint32) // Sets both CRC32C and SendCRC32C. + CloseWithError(err error) error + Attrs() *storage.ObjectAttrs + + embedToIncludeNewMethods() +} + +type Copier interface { + ObjectAttrs() *storage.ObjectAttrs + SetRewriteToken(string) + SetProgressFunc(func(uint64, uint64)) + SetDestinationKMSKeyName(string) + Run(context.Context) (*storage.ObjectAttrs, error) + + embedToIncludeNewMethods() +} + +type Composer interface { + ObjectAttrs() *storage.ObjectAttrs + Run(context.Context) (*storage.ObjectAttrs, error) + + embedToIncludeNewMethods() +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/desc.go b/vendor/github.com/prometheus/client_golang/prometheus/desc.go index 7b8827ffb..1d034f871 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/desc.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/desc.go @@ -93,7 +93,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // First add only the const label names and sort them... for labelName := range constLabels { if !checkLabelName(labelName) { - d.err = fmt.Errorf("%q is not a valid label name", labelName) + d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName) return d } labelNames = append(labelNames, labelName) @@ -115,7 +115,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // dimension with a different mix between preset and variable labels. for _, labelName := range variableLabels { if !checkLabelName(labelName) { - d.err = fmt.Errorf("%q is not a valid label name", labelName) + d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName) return d } labelNames = append(labelNames, "$"+labelName) diff --git a/vendor/github.com/prometheus/client_golang/prometheus/registry.go b/vendor/github.com/prometheus/client_golang/prometheus/registry.go index f98c81a86..b5e70b93f 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/registry.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/registry.go @@ -872,7 +872,13 @@ func checkMetricConsistency( h = hashAddByte(h, separatorByte) // Make sure label pairs are sorted. We depend on it for the consistency // check. - sort.Sort(labelPairSorter(dtoMetric.Label)) + if !sort.IsSorted(labelPairSorter(dtoMetric.Label)) { + // We cannot sort dtoMetric.Label in place as it is immutable by contract. + copiedLabels := make([]*dto.LabelPair, len(dtoMetric.Label)) + copy(copiedLabels, dtoMetric.Label) + sort.Sort(labelPairSorter(copiedLabels)) + dtoMetric.Label = copiedLabels + } for _, lp := range dtoMetric.Label { h = hashAdd(h, lp.GetName()) h = hashAddByte(h, separatorByte) @@ -903,8 +909,8 @@ func checkDescConsistency( } // Is the desc consistent with the content of the metric? - lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label)) - lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...) + lpsFromDesc := make([]*dto.LabelPair, len(desc.constLabelPairs), len(dtoMetric.Label)) + copy(lpsFromDesc, desc.constLabelPairs) for _, l := range desc.variableLabels { lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ Name: proto.String(l), diff --git a/vendor/github.com/prometheus/client_golang/prometheus/timer.go b/vendor/github.com/prometheus/client_golang/prometheus/timer.go index b8fc5f18c..8d5f10523 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/timer.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/timer.go @@ -39,13 +39,16 @@ func NewTimer(o Observer) *Timer { // ObserveDuration records the duration passed since the Timer was created with // NewTimer. It calls the Observe method of the Observer provided during -// construction with the duration in seconds as an argument. ObserveDuration is -// usually called with a defer statement. +// construction with the duration in seconds as an argument. The observed +// duration is also returned. ObserveDuration is usually called with a defer +// statement. // // Note that this method is only guaranteed to never observe negative durations // if used with Go1.9+. -func (t *Timer) ObserveDuration() { +func (t *Timer) ObserveDuration() time.Duration { + d := time.Since(t.begin) if t.observer != nil { - t.observer.Observe(time.Since(t.begin).Seconds()) + t.observer.Observe(d.Seconds()) } + return d } diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 3fe291884..f272e8f9f 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -97,6 +97,16 @@ type Transport struct { // to mean no limit. MaxHeaderListSize uint32 + // StrictMaxConcurrentStreams controls whether the server's + // SETTINGS_MAX_CONCURRENT_STREAMS should be respected + // globally. If false, new TCP connections are created to the + // server as needed to keep each under the per-connection + // SETTINGS_MAX_CONCURRENT_STREAMS limit. If true, the + // server's SETTINGS_MAX_CONCURRENT_STREAMS is interpreted as + // a global limit and callers of RoundTrip block when needed, + // waiting for their turn. + StrictMaxConcurrentStreams bool + // t1, if non-nil, is the standard library Transport using // this transport. Its settings are used (but not its // RoundTrip method, etc). @@ -711,8 +721,19 @@ func (cc *ClientConn) idleStateLocked() (st clientConnIdleState) { if cc.singleUse && cc.nextStreamID > 1 { return } - st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && - int64(cc.nextStreamID)+int64(cc.pendingRequests) < math.MaxInt32 + var maxConcurrentOkay bool + if cc.t.StrictMaxConcurrentStreams { + // We'll tell the caller we can take a new request to + // prevent the caller from dialing a new TCP + // connection, but then we'll block later before + // writing it. + maxConcurrentOkay = true + } else { + maxConcurrentOkay = int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) + } + + st.canTakeNewRequest = cc.goAway == nil && !cc.closed && !cc.closing && maxConcurrentOkay && + int64(cc.nextStreamID)+2*int64(cc.pendingRequests) < math.MaxInt32 st.freshConn = cc.nextStreamID == 1 && st.canTakeNewRequest return } diff --git a/vendor/golang.org/x/oauth2/internal/token.go b/vendor/golang.org/x/oauth2/internal/token.go index 1f607943f..a831b7746 100644 --- a/vendor/golang.org/x/oauth2/internal/token.go +++ b/vendor/golang.org/x/oauth2/internal/token.go @@ -136,6 +136,7 @@ var brokenAuthHeaderProviders = []string{ "https://accounts.zoho.com", "https://gitter.im/login/oauth/token", "https://openid-connect.onelogin.com/oidc", + "https://api.dailymotion.com/oauth/token", } // brokenAuthHeaderDomains lists broken providers that issue dynamic endpoints. diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go index 8f33ece7c..8d17873de 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_386.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include -m32 /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include -m32 /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build 386,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go index 70c1a2c12..b3d8ad79d 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_amd64.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include -m64 /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include -m64 /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build amd64,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go index 6e281d6b3..e092822fb 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build arm,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go index f9157e192..320696789 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_arm64.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include -fsigned-char /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include -fsigned-char /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build arm64,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go index 16714491a..6893a5bd0 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build mips,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go index 1270a1c90..40164cacd 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build mips64,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go index 93980be13..8a909738b 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mips64le.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build mips64le,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go index 0fc772619..8d7818422 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_mipsle.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build mipsle,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go index a5c5f3def..ec5bde3d5 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build ppc64,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go index 5470eadbf..bdbabdbcd 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_ppc64le.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build ppc64le,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go index a5d991915..473c74613 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_riscv64.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build riscv64,linux diff --git a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go index de0245a92..6eb7c257f 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_linux_s390x.go @@ -1,4 +1,4 @@ -// linux/mksysnum.pl -Wall -Werror -static -I/tmp/include -fsigned-char /tmp/include/asm/unistd.h +// go run linux/mksysnum.go -Wall -Werror -static -I/tmp/include -fsigned-char /tmp/include/asm/unistd.h // Code generated by the command above; see README.md. DO NOT EDIT. // +build s390x,linux diff --git a/vendor/google.golang.org/api/storage/v1/storage-gen.go b/vendor/google.golang.org/api/storage/v1/storage-gen.go index b8eabcc6f..606686f87 100644 --- a/vendor/google.golang.org/api/storage/v1/storage-gen.go +++ b/vendor/google.golang.org/api/storage/v1/storage-gen.go @@ -1,18 +1,8 @@ -// Copyright 2018 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Copyright 2018 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. -// AUTO-GENERATED CODE. DO NOT EDIT. +// Code generated file. DO NOT EDIT. // Package storage provides access to the Cloud Storage JSON API. // @@ -247,6 +237,9 @@ type Bucket struct { // Etag: HTTP 1.1 Entity tag for the bucket. Etag string `json:"etag,omitempty"` + // IamConfiguration: The bucket's IAM configuration. + IamConfiguration *BucketIamConfiguration `json:"iamConfiguration,omitempty"` + // Id: The ID of the bucket. For buckets, the id and name properties are // the same. Id string `json:"id,omitempty"` @@ -454,6 +447,69 @@ func (s *BucketEncryption) MarshalJSON() ([]byte, error) { return gensupport.MarshalJSON(raw, s.ForceSendFields, s.NullFields) } +// BucketIamConfiguration: The bucket's IAM configuration. +type BucketIamConfiguration struct { + BucketPolicyOnly *BucketIamConfigurationBucketPolicyOnly `json:"bucketPolicyOnly,omitempty"` + + // ForceSendFields is a list of field names (e.g. "BucketPolicyOnly") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` + + // NullFields is a list of field names (e.g. "BucketPolicyOnly") to + // include in API requests with the JSON null value. By default, fields + // with empty values are omitted from API requests. However, any field + // with an empty value appearing in NullFields will be sent to the + // server as null. It is an error if a field in this list has a + // non-empty value. This may be used to include null fields in Patch + // requests. + NullFields []string `json:"-"` +} + +func (s *BucketIamConfiguration) MarshalJSON() ([]byte, error) { + type NoMethod BucketIamConfiguration + raw := NoMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields, s.NullFields) +} + +type BucketIamConfigurationBucketPolicyOnly struct { + // Enabled: If set, access checks only use bucket-level IAM policies or + // above. + Enabled bool `json:"enabled,omitempty"` + + // LockedTime: The deadline time for changing + // iamConfiguration.bucketPolicyOnly.enabled from true to false in RFC + // 3339 format. iamConfiguration.bucketPolicyOnly.enabled may be changed + // from true to false until the locked time, after which the field is + // immutable. + LockedTime string `json:"lockedTime,omitempty"` + + // ForceSendFields is a list of field names (e.g. "Enabled") to + // unconditionally include in API requests. By default, fields with + // empty values are omitted from API requests. However, any non-pointer, + // non-interface field appearing in ForceSendFields will be sent to the + // server regardless of whether the field is empty or not. This may be + // used to include empty fields in Patch requests. + ForceSendFields []string `json:"-"` + + // NullFields is a list of field names (e.g. "Enabled") to include in + // API requests with the JSON null value. By default, fields with empty + // values are omitted from API requests. However, any field with an + // empty value appearing in NullFields will be sent to the server as + // null. It is an error if a field in this list has a non-empty value. + // This may be used to include null fields in Patch requests. + NullFields []string `json:"-"` +} + +func (s *BucketIamConfigurationBucketPolicyOnly) MarshalJSON() ([]byte, error) { + type NoMethod BucketIamConfigurationBucketPolicyOnly + raw := NoMethod(*s) + return gensupport.MarshalJSON(raw, s.ForceSendFields, s.NullFields) +} + // BucketLifecycle: The bucket's lifecycle configuration. See lifecycle // management for more information. type BucketLifecycle struct { diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go index 7e9e63c42..9521b50e9 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/annotations.pb.go @@ -1,14 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/annotations.proto -package annotations +package annotations // import "google.golang.org/genproto/googleapis/api/annotations" -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" - math "math" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -34,9 +32,11 @@ func init() { proto.RegisterExtension(E_Http) } -func init() { proto.RegisterFile("google/api/annotations.proto", fileDescriptor_c591c5aa9fb79aab) } +func init() { + proto.RegisterFile("google/api/annotations.proto", fileDescriptor_annotations_55609bb51d80951d) +} -var fileDescriptor_c591c5aa9fb79aab = []byte{ +var fileDescriptor_annotations_55609bb51d80951d = []byte{ // 208 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0xcf, 0xcf, 0x4f, 0xcf, 0x49, 0xd5, 0x4f, 0x2c, 0xc8, 0xd4, 0x4f, 0xcc, 0xcb, 0xcb, 0x2f, 0x49, 0x2c, 0xc9, 0xcc, diff --git a/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go b/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go index abe688ec7..1a8a27b65 100644 --- a/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/api/annotations/http.pb.go @@ -1,13 +1,11 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/api/http.proto -package annotations +package annotations // import "google.golang.org/genproto/googleapis/api/annotations" -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - math "math" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -44,17 +42,16 @@ func (m *Http) Reset() { *m = Http{} } func (m *Http) String() string { return proto.CompactTextString(m) } func (*Http) ProtoMessage() {} func (*Http) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9994be407cdcc9, []int{0} + return fileDescriptor_http_e457621dddd7365b, []int{0} } - func (m *Http) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Http.Unmarshal(m, b) } func (m *Http) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Http.Marshal(b, m, deterministic) } -func (m *Http) XXX_Merge(src proto.Message) { - xxx_messageInfo_Http.Merge(m, src) +func (dst *Http) XXX_Merge(src proto.Message) { + xxx_messageInfo_Http.Merge(dst, src) } func (m *Http) XXX_Size() int { return xxx_messageInfo_Http.Size(m) @@ -335,17 +332,16 @@ func (m *HttpRule) Reset() { *m = HttpRule{} } func (m *HttpRule) String() string { return proto.CompactTextString(m) } func (*HttpRule) ProtoMessage() {} func (*HttpRule) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9994be407cdcc9, []int{1} + return fileDescriptor_http_e457621dddd7365b, []int{1} } - func (m *HttpRule) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_HttpRule.Unmarshal(m, b) } func (m *HttpRule) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_HttpRule.Marshal(b, m, deterministic) } -func (m *HttpRule) XXX_Merge(src proto.Message) { - xxx_messageInfo_HttpRule.Merge(m, src) +func (dst *HttpRule) XXX_Merge(src proto.Message) { + xxx_messageInfo_HttpRule.Merge(dst, src) } func (m *HttpRule) XXX_Size() int { return xxx_messageInfo_HttpRule.Size(m) @@ -618,17 +614,16 @@ func (m *CustomHttpPattern) Reset() { *m = CustomHttpPattern{} } func (m *CustomHttpPattern) String() string { return proto.CompactTextString(m) } func (*CustomHttpPattern) ProtoMessage() {} func (*CustomHttpPattern) Descriptor() ([]byte, []int) { - return fileDescriptor_ff9994be407cdcc9, []int{2} + return fileDescriptor_http_e457621dddd7365b, []int{2} } - func (m *CustomHttpPattern) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CustomHttpPattern.Unmarshal(m, b) } func (m *CustomHttpPattern) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CustomHttpPattern.Marshal(b, m, deterministic) } -func (m *CustomHttpPattern) XXX_Merge(src proto.Message) { - xxx_messageInfo_CustomHttpPattern.Merge(m, src) +func (dst *CustomHttpPattern) XXX_Merge(src proto.Message) { + xxx_messageInfo_CustomHttpPattern.Merge(dst, src) } func (m *CustomHttpPattern) XXX_Size() int { return xxx_messageInfo_CustomHttpPattern.Size(m) @@ -659,9 +654,9 @@ func init() { proto.RegisterType((*CustomHttpPattern)(nil), "google.api.CustomHttpPattern") } -func init() { proto.RegisterFile("google/api/http.proto", fileDescriptor_ff9994be407cdcc9) } +func init() { proto.RegisterFile("google/api/http.proto", fileDescriptor_http_e457621dddd7365b) } -var fileDescriptor_ff9994be407cdcc9 = []byte{ +var fileDescriptor_http_e457621dddd7365b = []byte{ // 419 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x8e, 0xd3, 0x30, 0x10, 0x86, 0x49, 0x9b, 0x76, 0xdb, 0xe9, 0x82, 0x84, 0x59, 0x90, 0x85, 0x40, 0x54, 0xe5, 0x52, diff --git a/vendor/google.golang.org/genproto/googleapis/iam/v1/iam_policy.pb.go b/vendor/google.golang.org/genproto/googleapis/iam/v1/iam_policy.pb.go index 5e7893f3f..dfc87966b 100644 --- a/vendor/google.golang.org/genproto/googleapis/iam/v1/iam_policy.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/iam/v1/iam_policy.pb.go @@ -1,15 +1,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/iam/v1/iam_policy.proto -package iam +package iam // import "google.golang.org/genproto/googleapis/iam/v1" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "google.golang.org/genproto/googleapis/api/annotations" import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - _ "google.golang.org/genproto/googleapis/api/annotations" + context "golang.org/x/net/context" grpc "google.golang.org/grpc" - math "math" ) // Reference imports to suppress errors if they are not otherwise used. @@ -43,17 +44,16 @@ func (m *SetIamPolicyRequest) Reset() { *m = SetIamPolicyRequest{} } func (m *SetIamPolicyRequest) String() string { return proto.CompactTextString(m) } func (*SetIamPolicyRequest) ProtoMessage() {} func (*SetIamPolicyRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d2728eb97d748a32, []int{0} + return fileDescriptor_iam_policy_58547b5cf2e9d67a, []int{0} } - func (m *SetIamPolicyRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_SetIamPolicyRequest.Unmarshal(m, b) } func (m *SetIamPolicyRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_SetIamPolicyRequest.Marshal(b, m, deterministic) } -func (m *SetIamPolicyRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SetIamPolicyRequest.Merge(m, src) +func (dst *SetIamPolicyRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SetIamPolicyRequest.Merge(dst, src) } func (m *SetIamPolicyRequest) XXX_Size() int { return xxx_messageInfo_SetIamPolicyRequest.Size(m) @@ -93,17 +93,16 @@ func (m *GetIamPolicyRequest) Reset() { *m = GetIamPolicyRequest{} } func (m *GetIamPolicyRequest) String() string { return proto.CompactTextString(m) } func (*GetIamPolicyRequest) ProtoMessage() {} func (*GetIamPolicyRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d2728eb97d748a32, []int{1} + return fileDescriptor_iam_policy_58547b5cf2e9d67a, []int{1} } - func (m *GetIamPolicyRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_GetIamPolicyRequest.Unmarshal(m, b) } func (m *GetIamPolicyRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_GetIamPolicyRequest.Marshal(b, m, deterministic) } -func (m *GetIamPolicyRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetIamPolicyRequest.Merge(m, src) +func (dst *GetIamPolicyRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetIamPolicyRequest.Merge(dst, src) } func (m *GetIamPolicyRequest) XXX_Size() int { return xxx_messageInfo_GetIamPolicyRequest.Size(m) @@ -141,17 +140,16 @@ func (m *TestIamPermissionsRequest) Reset() { *m = TestIamPermissionsReq func (m *TestIamPermissionsRequest) String() string { return proto.CompactTextString(m) } func (*TestIamPermissionsRequest) ProtoMessage() {} func (*TestIamPermissionsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_d2728eb97d748a32, []int{2} + return fileDescriptor_iam_policy_58547b5cf2e9d67a, []int{2} } - func (m *TestIamPermissionsRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TestIamPermissionsRequest.Unmarshal(m, b) } func (m *TestIamPermissionsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TestIamPermissionsRequest.Marshal(b, m, deterministic) } -func (m *TestIamPermissionsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_TestIamPermissionsRequest.Merge(m, src) +func (dst *TestIamPermissionsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TestIamPermissionsRequest.Merge(dst, src) } func (m *TestIamPermissionsRequest) XXX_Size() int { return xxx_messageInfo_TestIamPermissionsRequest.Size(m) @@ -190,17 +188,16 @@ func (m *TestIamPermissionsResponse) Reset() { *m = TestIamPermissionsRe func (m *TestIamPermissionsResponse) String() string { return proto.CompactTextString(m) } func (*TestIamPermissionsResponse) ProtoMessage() {} func (*TestIamPermissionsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_d2728eb97d748a32, []int{3} + return fileDescriptor_iam_policy_58547b5cf2e9d67a, []int{3} } - func (m *TestIamPermissionsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TestIamPermissionsResponse.Unmarshal(m, b) } func (m *TestIamPermissionsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TestIamPermissionsResponse.Marshal(b, m, deterministic) } -func (m *TestIamPermissionsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_TestIamPermissionsResponse.Merge(m, src) +func (dst *TestIamPermissionsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TestIamPermissionsResponse.Merge(dst, src) } func (m *TestIamPermissionsResponse) XXX_Size() int { return xxx_messageInfo_TestIamPermissionsResponse.Size(m) @@ -225,38 +222,6 @@ func init() { proto.RegisterType((*TestIamPermissionsResponse)(nil), "google.iam.v1.TestIamPermissionsResponse") } -func init() { proto.RegisterFile("google/iam/v1/iam_policy.proto", fileDescriptor_d2728eb97d748a32) } - -var fileDescriptor_d2728eb97d748a32 = []byte{ - // 411 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4b, 0xcf, 0xcf, 0x4f, - 0xcf, 0x49, 0xd5, 0xcf, 0x4c, 0xcc, 0xd5, 0x2f, 0x33, 0x04, 0x51, 0xf1, 0x05, 0xf9, 0x39, 0x99, - 0xc9, 0x95, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xbc, 0x10, 0x79, 0xbd, 0xcc, 0xc4, 0x5c, - 0xbd, 0x32, 0x43, 0x29, 0x19, 0xa8, 0xf2, 0xc4, 0x82, 0x4c, 0xfd, 0xc4, 0xbc, 0xbc, 0xfc, 0x92, - 0xc4, 0x92, 0xcc, 0xfc, 0xbc, 0x62, 0x88, 0x62, 0x29, 0x29, 0x54, 0xc3, 0x90, 0x0d, 0x52, 0x4a, - 0xe0, 0x12, 0x0e, 0x4e, 0x2d, 0xf1, 0x4c, 0xcc, 0x0d, 0x00, 0x8b, 0x06, 0xa5, 0x16, 0x96, 0xa6, - 0x16, 0x97, 0x08, 0x49, 0x71, 0x71, 0x14, 0xa5, 0x16, 0xe7, 0x97, 0x16, 0x25, 0xa7, 0x4a, 0x30, - 0x2a, 0x30, 0x6a, 0x70, 0x06, 0xc1, 0xf9, 0x42, 0xba, 0x5c, 0x6c, 0x10, 0x23, 0x24, 0x98, 0x14, - 0x18, 0x35, 0xb8, 0x8d, 0x44, 0xf5, 0x50, 0x1c, 0xa3, 0x07, 0x35, 0x09, 0xaa, 0x48, 0xc9, 0x90, - 0x4b, 0xd8, 0x9d, 0x34, 0x1b, 0x94, 0x22, 0xb9, 0x24, 0x43, 0x52, 0x8b, 0xc1, 0x7a, 0x52, 0x8b, - 0x72, 0x33, 0x8b, 0x8b, 0x41, 0x9e, 0x21, 0xc6, 0x69, 0x0a, 0x5c, 0xdc, 0x05, 0x08, 0x1d, 0x12, - 0x4c, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0xc8, 0x42, 0x4a, 0x76, 0x5c, 0x52, 0xd8, 0x8c, 0x2e, 0x2e, - 0xc8, 0xcf, 0x2b, 0xc6, 0xd0, 0xcf, 0x88, 0xa1, 0xdf, 0x68, 0x0a, 0x33, 0x17, 0xa7, 0xa7, 0xa3, - 0x2f, 0xc4, 0x2f, 0x42, 0x25, 0x5c, 0x3c, 0xc8, 0xa1, 0x27, 0xa4, 0x84, 0x16, 0x14, 0x58, 0x82, - 0x56, 0x0a, 0x7b, 0x70, 0x29, 0x69, 0x36, 0x5d, 0x7e, 0x32, 0x99, 0x49, 0x59, 0x49, 0x0e, 0x14, - 0x45, 0xd5, 0x30, 0x1f, 0xd9, 0x6a, 0x69, 0xd5, 0x5a, 0x15, 0x23, 0x99, 0x62, 0xc5, 0xa8, 0x05, - 0xb2, 0xd5, 0x1d, 0x9f, 0xad, 0xee, 0x54, 0xb1, 0x35, 0x1d, 0xcd, 0xd6, 0x59, 0x8c, 0x5c, 0x42, - 0x98, 0x41, 0x27, 0xa4, 0x81, 0x66, 0x30, 0xce, 0x88, 0x93, 0xd2, 0x24, 0x42, 0x25, 0x24, 0x1e, - 0x94, 0xf4, 0xc1, 0xce, 0xd2, 0x54, 0x52, 0xc1, 0x74, 0x56, 0x09, 0x86, 0x2e, 0x2b, 0x46, 0x2d, - 0xa7, 0x36, 0x46, 0x2e, 0xc1, 0xe4, 0xfc, 0x5c, 0x54, 0x1b, 0x9c, 0xf8, 0xe0, 0x1e, 0x08, 0x00, - 0x25, 0xf6, 0x00, 0xc6, 0x28, 0x03, 0xa8, 0x82, 0xf4, 0xfc, 0x9c, 0xc4, 0xbc, 0x74, 0xbd, 0xfc, - 0xa2, 0x74, 0xfd, 0xf4, 0xd4, 0x3c, 0x70, 0x56, 0xd0, 0x87, 0x48, 0x25, 0x16, 0x64, 0x16, 0x43, - 0x73, 0x8a, 0x75, 0x66, 0x62, 0xee, 0x0f, 0x46, 0xc6, 0x55, 0x4c, 0xc2, 0xee, 0x10, 0x5d, 0xce, - 0x39, 0xf9, 0xa5, 0x29, 0x7a, 0x9e, 0x89, 0xb9, 0x7a, 0x61, 0x86, 0xa7, 0x60, 0xa2, 0x31, 0x60, - 0xd1, 0x18, 0xcf, 0xc4, 0xdc, 0x98, 0x30, 0xc3, 0x24, 0x36, 0xb0, 0x59, 0xc6, 0x80, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xea, 0x62, 0x8f, 0x22, 0xc1, 0x03, 0x00, 0x00, -} - // Reference imports to suppress errors if they are not otherwise used. var _ context.Context var _ grpc.ClientConn @@ -410,3 +375,37 @@ var _IAMPolicy_serviceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "google/iam/v1/iam_policy.proto", } + +func init() { + proto.RegisterFile("google/iam/v1/iam_policy.proto", fileDescriptor_iam_policy_58547b5cf2e9d67a) +} + +var fileDescriptor_iam_policy_58547b5cf2e9d67a = []byte{ + // 411 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4b, 0xcf, 0xcf, 0x4f, + 0xcf, 0x49, 0xd5, 0xcf, 0x4c, 0xcc, 0xd5, 0x2f, 0x33, 0x04, 0x51, 0xf1, 0x05, 0xf9, 0x39, 0x99, + 0xc9, 0x95, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xbc, 0x10, 0x79, 0xbd, 0xcc, 0xc4, 0x5c, + 0xbd, 0x32, 0x43, 0x29, 0x19, 0xa8, 0xf2, 0xc4, 0x82, 0x4c, 0xfd, 0xc4, 0xbc, 0xbc, 0xfc, 0x92, + 0xc4, 0x92, 0xcc, 0xfc, 0xbc, 0x62, 0x88, 0x62, 0x29, 0x29, 0x54, 0xc3, 0x90, 0x0d, 0x52, 0x4a, + 0xe0, 0x12, 0x0e, 0x4e, 0x2d, 0xf1, 0x4c, 0xcc, 0x0d, 0x00, 0x8b, 0x06, 0xa5, 0x16, 0x96, 0xa6, + 0x16, 0x97, 0x08, 0x49, 0x71, 0x71, 0x14, 0xa5, 0x16, 0xe7, 0x97, 0x16, 0x25, 0xa7, 0x4a, 0x30, + 0x2a, 0x30, 0x6a, 0x70, 0x06, 0xc1, 0xf9, 0x42, 0xba, 0x5c, 0x6c, 0x10, 0x23, 0x24, 0x98, 0x14, + 0x18, 0x35, 0xb8, 0x8d, 0x44, 0xf5, 0x50, 0x1c, 0xa3, 0x07, 0x35, 0x09, 0xaa, 0x48, 0xc9, 0x90, + 0x4b, 0xd8, 0x9d, 0x34, 0x1b, 0x94, 0x22, 0xb9, 0x24, 0x43, 0x52, 0x8b, 0xc1, 0x7a, 0x52, 0x8b, + 0x72, 0x33, 0x8b, 0x8b, 0x41, 0x9e, 0x21, 0xc6, 0x69, 0x0a, 0x5c, 0xdc, 0x05, 0x08, 0x1d, 0x12, + 0x4c, 0x0a, 0xcc, 0x1a, 0x9c, 0x41, 0xc8, 0x42, 0x4a, 0x76, 0x5c, 0x52, 0xd8, 0x8c, 0x2e, 0x2e, + 0xc8, 0xcf, 0x2b, 0xc6, 0xd0, 0xcf, 0x88, 0xa1, 0xdf, 0x68, 0x0a, 0x33, 0x17, 0xa7, 0xa7, 0xa3, + 0x2f, 0xc4, 0x2f, 0x42, 0x25, 0x5c, 0x3c, 0xc8, 0xa1, 0x27, 0xa4, 0x84, 0x16, 0x14, 0x58, 0x82, + 0x56, 0x0a, 0x7b, 0x70, 0x29, 0x69, 0x36, 0x5d, 0x7e, 0x32, 0x99, 0x49, 0x59, 0x49, 0x0e, 0x14, + 0x45, 0xd5, 0x30, 0x1f, 0xd9, 0x6a, 0x69, 0xd5, 0x5a, 0x15, 0x23, 0x99, 0x62, 0xc5, 0xa8, 0x05, + 0xb2, 0xd5, 0x1d, 0x9f, 0xad, 0xee, 0x54, 0xb1, 0x35, 0x1d, 0xcd, 0xd6, 0x59, 0x8c, 0x5c, 0x42, + 0x98, 0x41, 0x27, 0xa4, 0x81, 0x66, 0x30, 0xce, 0x88, 0x93, 0xd2, 0x24, 0x42, 0x25, 0x24, 0x1e, + 0x94, 0xf4, 0xc1, 0xce, 0xd2, 0x54, 0x52, 0xc1, 0x74, 0x56, 0x09, 0x86, 0x2e, 0x2b, 0x46, 0x2d, + 0xa7, 0x36, 0x46, 0x2e, 0xc1, 0xe4, 0xfc, 0x5c, 0x54, 0x1b, 0x9c, 0xf8, 0xe0, 0x1e, 0x08, 0x00, + 0x25, 0xf6, 0x00, 0xc6, 0x28, 0x03, 0xa8, 0x82, 0xf4, 0xfc, 0x9c, 0xc4, 0xbc, 0x74, 0xbd, 0xfc, + 0xa2, 0x74, 0xfd, 0xf4, 0xd4, 0x3c, 0x70, 0x56, 0xd0, 0x87, 0x48, 0x25, 0x16, 0x64, 0x16, 0x43, + 0x73, 0x8a, 0x75, 0x66, 0x62, 0xee, 0x0f, 0x46, 0xc6, 0x55, 0x4c, 0xc2, 0xee, 0x10, 0x5d, 0xce, + 0x39, 0xf9, 0xa5, 0x29, 0x7a, 0x9e, 0x89, 0xb9, 0x7a, 0x61, 0x86, 0xa7, 0x60, 0xa2, 0x31, 0x60, + 0xd1, 0x18, 0xcf, 0xc4, 0xdc, 0x98, 0x30, 0xc3, 0x24, 0x36, 0xb0, 0x59, 0xc6, 0x80, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xea, 0x62, 0x8f, 0x22, 0xc1, 0x03, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/genproto/googleapis/iam/v1/policy.pb.go b/vendor/google.golang.org/genproto/googleapis/iam/v1/policy.pb.go index 5d966b067..99dd75f26 100644 --- a/vendor/google.golang.org/genproto/googleapis/iam/v1/policy.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/iam/v1/policy.pb.go @@ -1,14 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/iam/v1/policy.proto -package iam +package iam // import "google.golang.org/genproto/googleapis/iam/v1" -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - _ "google.golang.org/genproto/googleapis/api/annotations" - math "math" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import _ "google.golang.org/genproto/googleapis/api/annotations" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -38,7 +36,6 @@ var BindingDelta_Action_name = map[int32]string{ 1: "ADD", 2: "REMOVE", } - var BindingDelta_Action_value = map[string]int32{ "ACTION_UNSPECIFIED": 0, "ADD": 1, @@ -48,9 +45,8 @@ var BindingDelta_Action_value = map[string]int32{ func (x BindingDelta_Action) String() string { return proto.EnumName(BindingDelta_Action_name, int32(x)) } - func (BindingDelta_Action) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_a3cd40b8a66b2a99, []int{3, 0} + return fileDescriptor_policy_6ba2a3dcbcdd909c, []int{3, 0} } // Defines an Identity and Access Management (IAM) policy. It is used to @@ -111,17 +107,16 @@ func (m *Policy) Reset() { *m = Policy{} } func (m *Policy) String() string { return proto.CompactTextString(m) } func (*Policy) ProtoMessage() {} func (*Policy) Descriptor() ([]byte, []int) { - return fileDescriptor_a3cd40b8a66b2a99, []int{0} + return fileDescriptor_policy_6ba2a3dcbcdd909c, []int{0} } - func (m *Policy) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Policy.Unmarshal(m, b) } func (m *Policy) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Policy.Marshal(b, m, deterministic) } -func (m *Policy) XXX_Merge(src proto.Message) { - xxx_messageInfo_Policy.Merge(m, src) +func (dst *Policy) XXX_Merge(src proto.Message) { + xxx_messageInfo_Policy.Merge(dst, src) } func (m *Policy) XXX_Size() int { return xxx_messageInfo_Policy.Size(m) @@ -192,17 +187,16 @@ func (m *Binding) Reset() { *m = Binding{} } func (m *Binding) String() string { return proto.CompactTextString(m) } func (*Binding) ProtoMessage() {} func (*Binding) Descriptor() ([]byte, []int) { - return fileDescriptor_a3cd40b8a66b2a99, []int{1} + return fileDescriptor_policy_6ba2a3dcbcdd909c, []int{1} } - func (m *Binding) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Binding.Unmarshal(m, b) } func (m *Binding) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Binding.Marshal(b, m, deterministic) } -func (m *Binding) XXX_Merge(src proto.Message) { - xxx_messageInfo_Binding.Merge(m, src) +func (dst *Binding) XXX_Merge(src proto.Message) { + xxx_messageInfo_Binding.Merge(dst, src) } func (m *Binding) XXX_Size() int { return xxx_messageInfo_Binding.Size(m) @@ -240,17 +234,16 @@ func (m *PolicyDelta) Reset() { *m = PolicyDelta{} } func (m *PolicyDelta) String() string { return proto.CompactTextString(m) } func (*PolicyDelta) ProtoMessage() {} func (*PolicyDelta) Descriptor() ([]byte, []int) { - return fileDescriptor_a3cd40b8a66b2a99, []int{2} + return fileDescriptor_policy_6ba2a3dcbcdd909c, []int{2} } - func (m *PolicyDelta) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PolicyDelta.Unmarshal(m, b) } func (m *PolicyDelta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_PolicyDelta.Marshal(b, m, deterministic) } -func (m *PolicyDelta) XXX_Merge(src proto.Message) { - xxx_messageInfo_PolicyDelta.Merge(m, src) +func (dst *PolicyDelta) XXX_Merge(src proto.Message) { + xxx_messageInfo_PolicyDelta.Merge(dst, src) } func (m *PolicyDelta) XXX_Size() int { return xxx_messageInfo_PolicyDelta.Size(m) @@ -291,17 +284,16 @@ func (m *BindingDelta) Reset() { *m = BindingDelta{} } func (m *BindingDelta) String() string { return proto.CompactTextString(m) } func (*BindingDelta) ProtoMessage() {} func (*BindingDelta) Descriptor() ([]byte, []int) { - return fileDescriptor_a3cd40b8a66b2a99, []int{3} + return fileDescriptor_policy_6ba2a3dcbcdd909c, []int{3} } - func (m *BindingDelta) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BindingDelta.Unmarshal(m, b) } func (m *BindingDelta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BindingDelta.Marshal(b, m, deterministic) } -func (m *BindingDelta) XXX_Merge(src proto.Message) { - xxx_messageInfo_BindingDelta.Merge(m, src) +func (dst *BindingDelta) XXX_Merge(src proto.Message) { + xxx_messageInfo_BindingDelta.Merge(dst, src) } func (m *BindingDelta) XXX_Size() int { return xxx_messageInfo_BindingDelta.Size(m) @@ -334,16 +326,16 @@ func (m *BindingDelta) GetMember() string { } func init() { - proto.RegisterEnum("google.iam.v1.BindingDelta_Action", BindingDelta_Action_name, BindingDelta_Action_value) proto.RegisterType((*Policy)(nil), "google.iam.v1.Policy") proto.RegisterType((*Binding)(nil), "google.iam.v1.Binding") proto.RegisterType((*PolicyDelta)(nil), "google.iam.v1.PolicyDelta") proto.RegisterType((*BindingDelta)(nil), "google.iam.v1.BindingDelta") + proto.RegisterEnum("google.iam.v1.BindingDelta_Action", BindingDelta_Action_name, BindingDelta_Action_value) } -func init() { proto.RegisterFile("google/iam/v1/policy.proto", fileDescriptor_a3cd40b8a66b2a99) } +func init() { proto.RegisterFile("google/iam/v1/policy.proto", fileDescriptor_policy_6ba2a3dcbcdd909c) } -var fileDescriptor_a3cd40b8a66b2a99 = []byte{ +var fileDescriptor_policy_6ba2a3dcbcdd909c = []byte{ // 403 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0x4d, 0xab, 0x13, 0x31, 0x14, 0x35, 0xed, 0x73, 0x6a, 0xef, 0xfb, 0xa0, 0x46, 0x28, 0xc3, 0xd3, 0x45, 0x99, 0x55, 0x57, diff --git a/vendor/google.golang.org/genproto/googleapis/rpc/code/code.pb.go b/vendor/google.golang.org/genproto/googleapis/rpc/code/code.pb.go new file mode 100644 index 000000000..410e374c8 --- /dev/null +++ b/vendor/google.golang.org/genproto/googleapis/rpc/code/code.pb.go @@ -0,0 +1,246 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: google/rpc/code.proto + +package code // import "google.golang.org/genproto/googleapis/rpc/code" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// The canonical error codes for Google APIs. +// +// +// Sometimes multiple error codes may apply. Services should return +// the most specific error code that applies. For example, prefer +// `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply. +// Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`. +type Code int32 + +const ( + // Not an error; returned on success + // + // HTTP Mapping: 200 OK + Code_OK Code = 0 + // The operation was cancelled, typically by the caller. + // + // HTTP Mapping: 499 Client Closed Request + Code_CANCELLED Code = 1 + // Unknown error. For example, this error may be returned when + // a `Status` value received from another address space belongs to + // an error space that is not known in this address space. Also + // errors raised by APIs that do not return enough error information + // may be converted to this error. + // + // HTTP Mapping: 500 Internal Server Error + Code_UNKNOWN Code = 2 + // The client specified an invalid argument. Note that this differs + // from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments + // that are problematic regardless of the state of the system + // (e.g., a malformed file name). + // + // HTTP Mapping: 400 Bad Request + Code_INVALID_ARGUMENT Code = 3 + // The deadline expired before the operation could complete. For operations + // that change the state of the system, this error may be returned + // even if the operation has completed successfully. For example, a + // successful response from a server could have been delayed long + // enough for the deadline to expire. + // + // HTTP Mapping: 504 Gateway Timeout + Code_DEADLINE_EXCEEDED Code = 4 + // Some requested entity (e.g., file or directory) was not found. + // + // Note to server developers: if a request is denied for an entire class + // of users, such as gradual feature rollout or undocumented whitelist, + // `NOT_FOUND` may be used. If a request is denied for some users within + // a class of users, such as user-based access control, `PERMISSION_DENIED` + // must be used. + // + // HTTP Mapping: 404 Not Found + Code_NOT_FOUND Code = 5 + // The entity that a client attempted to create (e.g., file or directory) + // already exists. + // + // HTTP Mapping: 409 Conflict + Code_ALREADY_EXISTS Code = 6 + // The caller does not have permission to execute the specified + // operation. `PERMISSION_DENIED` must not be used for rejections + // caused by exhausting some resource (use `RESOURCE_EXHAUSTED` + // instead for those errors). `PERMISSION_DENIED` must not be + // used if the caller can not be identified (use `UNAUTHENTICATED` + // instead for those errors). This error code does not imply the + // request is valid or the requested entity exists or satisfies + // other pre-conditions. + // + // HTTP Mapping: 403 Forbidden + Code_PERMISSION_DENIED Code = 7 + // The request does not have valid authentication credentials for the + // operation. + // + // HTTP Mapping: 401 Unauthorized + Code_UNAUTHENTICATED Code = 16 + // Some resource has been exhausted, perhaps a per-user quota, or + // perhaps the entire file system is out of space. + // + // HTTP Mapping: 429 Too Many Requests + Code_RESOURCE_EXHAUSTED Code = 8 + // The operation was rejected because the system is not in a state + // required for the operation's execution. For example, the directory + // to be deleted is non-empty, an rmdir operation is applied to + // a non-directory, etc. + // + // Service implementors can use the following guidelines to decide + // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`: + // (a) Use `UNAVAILABLE` if the client can retry just the failing call. + // (b) Use `ABORTED` if the client should retry at a higher level + // (e.g., when a client-specified test-and-set fails, indicating the + // client should restart a read-modify-write sequence). + // (c) Use `FAILED_PRECONDITION` if the client should not retry until + // the system state has been explicitly fixed. E.g., if an "rmdir" + // fails because the directory is non-empty, `FAILED_PRECONDITION` + // should be returned since the client should not retry unless + // the files are deleted from the directory. + // + // HTTP Mapping: 400 Bad Request + Code_FAILED_PRECONDITION Code = 9 + // The operation was aborted, typically due to a concurrency issue such as + // a sequencer check failure or transaction abort. + // + // See the guidelines above for deciding between `FAILED_PRECONDITION`, + // `ABORTED`, and `UNAVAILABLE`. + // + // HTTP Mapping: 409 Conflict + Code_ABORTED Code = 10 + // The operation was attempted past the valid range. E.g., seeking or + // reading past end-of-file. + // + // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may + // be fixed if the system state changes. For example, a 32-bit file + // system will generate `INVALID_ARGUMENT` if asked to read at an + // offset that is not in the range [0,2^32-1], but it will generate + // `OUT_OF_RANGE` if asked to read from an offset past the current + // file size. + // + // There is a fair bit of overlap between `FAILED_PRECONDITION` and + // `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific + // error) when it applies so that callers who are iterating through + // a space can easily look for an `OUT_OF_RANGE` error to detect when + // they are done. + // + // HTTP Mapping: 400 Bad Request + Code_OUT_OF_RANGE Code = 11 + // The operation is not implemented or is not supported/enabled in this + // service. + // + // HTTP Mapping: 501 Not Implemented + Code_UNIMPLEMENTED Code = 12 + // Internal errors. This means that some invariants expected by the + // underlying system have been broken. This error code is reserved + // for serious errors. + // + // HTTP Mapping: 500 Internal Server Error + Code_INTERNAL Code = 13 + // The service is currently unavailable. This is most likely a + // transient condition, which can be corrected by retrying with + // a backoff. + // + // See the guidelines above for deciding between `FAILED_PRECONDITION`, + // `ABORTED`, and `UNAVAILABLE`. + // + // HTTP Mapping: 503 Service Unavailable + Code_UNAVAILABLE Code = 14 + // Unrecoverable data loss or corruption. + // + // HTTP Mapping: 500 Internal Server Error + Code_DATA_LOSS Code = 15 +) + +var Code_name = map[int32]string{ + 0: "OK", + 1: "CANCELLED", + 2: "UNKNOWN", + 3: "INVALID_ARGUMENT", + 4: "DEADLINE_EXCEEDED", + 5: "NOT_FOUND", + 6: "ALREADY_EXISTS", + 7: "PERMISSION_DENIED", + 16: "UNAUTHENTICATED", + 8: "RESOURCE_EXHAUSTED", + 9: "FAILED_PRECONDITION", + 10: "ABORTED", + 11: "OUT_OF_RANGE", + 12: "UNIMPLEMENTED", + 13: "INTERNAL", + 14: "UNAVAILABLE", + 15: "DATA_LOSS", +} +var Code_value = map[string]int32{ + "OK": 0, + "CANCELLED": 1, + "UNKNOWN": 2, + "INVALID_ARGUMENT": 3, + "DEADLINE_EXCEEDED": 4, + "NOT_FOUND": 5, + "ALREADY_EXISTS": 6, + "PERMISSION_DENIED": 7, + "UNAUTHENTICATED": 16, + "RESOURCE_EXHAUSTED": 8, + "FAILED_PRECONDITION": 9, + "ABORTED": 10, + "OUT_OF_RANGE": 11, + "UNIMPLEMENTED": 12, + "INTERNAL": 13, + "UNAVAILABLE": 14, + "DATA_LOSS": 15, +} + +func (x Code) String() string { + return proto.EnumName(Code_name, int32(x)) +} +func (Code) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_code_932ba152e0df0902, []int{0} +} + +func init() { + proto.RegisterEnum("google.rpc.Code", Code_name, Code_value) +} + +func init() { proto.RegisterFile("google/rpc/code.proto", fileDescriptor_code_932ba152e0df0902) } + +var fileDescriptor_code_932ba152e0df0902 = []byte{ + // 362 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x44, 0x51, 0xcd, 0x6e, 0x93, 0x31, + 0x10, 0xa4, 0x69, 0x49, 0x9b, 0xcd, 0xdf, 0xd6, 0xa5, 0xf0, 0x0e, 0x1c, 0x92, 0x43, 0x8f, 0x9c, + 0x36, 0x9f, 0x37, 0xad, 0x55, 0x67, 0xfd, 0xc9, 0x3f, 0x25, 0x70, 0xb1, 0x4a, 0x1a, 0x7d, 0x42, + 0x2a, 0x75, 0xf4, 0xc1, 0x13, 0xf1, 0x12, 0xbc, 0x1e, 0x72, 0x8b, 0xe8, 0xc5, 0x87, 0x99, 0xf1, + 0xee, 0xce, 0x0c, 0x5c, 0x76, 0xa5, 0x74, 0x8f, 0xfb, 0x65, 0x7f, 0xd8, 0x2d, 0x77, 0xe5, 0x61, + 0xbf, 0x38, 0xf4, 0xe5, 0x57, 0x51, 0xf0, 0x02, 0x2f, 0xfa, 0xc3, 0xee, 0xe3, 0x9f, 0x01, 0x9c, + 0x34, 0xe5, 0x61, 0xaf, 0x86, 0x30, 0x70, 0xb7, 0xf8, 0x46, 0x4d, 0x61, 0xd4, 0x90, 0x34, 0x6c, + 0x2d, 0x6b, 0x3c, 0x52, 0x63, 0x38, 0x4d, 0x72, 0x2b, 0xee, 0xb3, 0xe0, 0x40, 0xbd, 0x03, 0x34, + 0x72, 0x47, 0xd6, 0xe8, 0x4c, 0xfe, 0x3a, 0x6d, 0x58, 0x22, 0x1e, 0xab, 0x4b, 0x38, 0xd7, 0x4c, + 0xda, 0x1a, 0xe1, 0xcc, 0xdb, 0x86, 0x59, 0xb3, 0xc6, 0x93, 0x3a, 0x48, 0x5c, 0xcc, 0x6b, 0x97, + 0x44, 0xe3, 0x5b, 0xa5, 0x60, 0x46, 0xd6, 0x33, 0xe9, 0x2f, 0x99, 0xb7, 0x26, 0xc4, 0x80, 0xc3, + 0xfa, 0xb3, 0x65, 0xbf, 0x31, 0x21, 0x18, 0x27, 0x59, 0xb3, 0x18, 0xd6, 0x78, 0xaa, 0x2e, 0x60, + 0x9e, 0x84, 0x52, 0xbc, 0x61, 0x89, 0xa6, 0xa1, 0xc8, 0x1a, 0x51, 0xbd, 0x07, 0xe5, 0x39, 0xb8, + 0xe4, 0x9b, 0xba, 0xe5, 0x86, 0x52, 0xa8, 0xf8, 0x99, 0xfa, 0x00, 0x17, 0x6b, 0x32, 0x96, 0x75, + 0x6e, 0x3d, 0x37, 0x4e, 0xb4, 0x89, 0xc6, 0x09, 0x8e, 0xea, 0xe5, 0xb4, 0x72, 0xbe, 0xaa, 0x40, + 0x21, 0x4c, 0x5c, 0x8a, 0xd9, 0xad, 0xb3, 0x27, 0xb9, 0x66, 0x1c, 0xab, 0x73, 0x98, 0x26, 0x31, + 0x9b, 0xd6, 0x72, 0xb5, 0xc1, 0x1a, 0x27, 0x6a, 0x02, 0x67, 0x46, 0x22, 0x7b, 0x21, 0x8b, 0x53, + 0x35, 0x87, 0x71, 0x12, 0xba, 0x23, 0x63, 0x69, 0x65, 0x19, 0x67, 0xd5, 0x90, 0xa6, 0x48, 0xd9, + 0xba, 0x10, 0x70, 0xbe, 0xda, 0xc2, 0x6c, 0x57, 0x7e, 0x2c, 0x5e, 0xb3, 0x5c, 0x8d, 0x6a, 0x90, + 0x6d, 0x8d, 0xb8, 0x3d, 0xfa, 0x7a, 0xf5, 0x8f, 0xe8, 0xca, 0xe3, 0xfd, 0x53, 0xb7, 0x28, 0x7d, + 0xb7, 0xec, 0xf6, 0x4f, 0xcf, 0x05, 0x2c, 0x5f, 0xa8, 0xfb, 0xc3, 0xf7, 0x9f, 0xff, 0xab, 0xf9, + 0x54, 0x9f, 0xdf, 0x83, 0x63, 0xdf, 0x36, 0xdf, 0x86, 0xcf, 0xaa, 0xab, 0xbf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0x8e, 0x97, 0x77, 0xc2, 0xbf, 0x01, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go index d13bcbaf4..7bfe37a3d 100644 --- a/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go +++ b/vendor/google.golang.org/genproto/googleapis/rpc/status/status.pb.go @@ -1,14 +1,12 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: google/rpc/status.proto -package status +package status // import "google.golang.org/genproto/googleapis/rpc/status" -import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - any "github.com/golang/protobuf/ptypes/any" - math "math" -) +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import any "github.com/golang/protobuf/ptypes/any" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal @@ -92,17 +90,16 @@ func (m *Status) Reset() { *m = Status{} } func (m *Status) String() string { return proto.CompactTextString(m) } func (*Status) ProtoMessage() {} func (*Status) Descriptor() ([]byte, []int) { - return fileDescriptor_24d244abaf643bfe, []int{0} + return fileDescriptor_status_c6e4de62dcdf2edf, []int{0} } - func (m *Status) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Status.Unmarshal(m, b) } func (m *Status) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Status.Marshal(b, m, deterministic) } -func (m *Status) XXX_Merge(src proto.Message) { - xxx_messageInfo_Status.Merge(m, src) +func (dst *Status) XXX_Merge(src proto.Message) { + xxx_messageInfo_Status.Merge(dst, src) } func (m *Status) XXX_Size() int { return xxx_messageInfo_Status.Size(m) @@ -138,9 +135,9 @@ func init() { proto.RegisterType((*Status)(nil), "google.rpc.Status") } -func init() { proto.RegisterFile("google/rpc/status.proto", fileDescriptor_24d244abaf643bfe) } +func init() { proto.RegisterFile("google/rpc/status.proto", fileDescriptor_status_c6e4de62dcdf2edf) } -var fileDescriptor_24d244abaf643bfe = []byte{ +var fileDescriptor_status_c6e4de62dcdf2edf = []byte{ // 209 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x12, 0x4f, 0xcf, 0xcf, 0x4f, 0xcf, 0x49, 0xd5, 0x2f, 0x2a, 0x48, 0xd6, 0x2f, 0x2e, 0x49, 0x2c, 0x29, 0x2d, 0xd6, 0x2b, 0x28, diff --git a/vendor/google.golang.org/grpc/balancer.go b/vendor/google.golang.org/grpc/balancer.go index 5aeb646d1..a78e702ba 100644 --- a/vendor/google.golang.org/grpc/balancer.go +++ b/vendor/google.golang.org/grpc/balancer.go @@ -19,10 +19,10 @@ package grpc import ( + "context" "net" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index ee1703f03..1bf46aafe 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -21,11 +21,11 @@ package balancer import ( + "context" "errors" "net" "strings" - "golang.org/x/net/context" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/metadata" @@ -94,6 +94,9 @@ type NewSubConnOptions struct { // SubConn. If it's nil, the original creds from grpc DialOptions will be // used. CredsBundle credentials.Bundle + // HealthCheckEnabled indicates whether health check service should be + // enabled on this SubConn + HealthCheckEnabled bool } // ClientConn represents a gRPC ClientConn. diff --git a/vendor/google.golang.org/grpc/balancer/base/balancer.go b/vendor/google.golang.org/grpc/balancer/base/balancer.go index 23d13511b..5f55b274f 100644 --- a/vendor/google.golang.org/grpc/balancer/base/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/base/balancer.go @@ -19,7 +19,8 @@ package base import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" @@ -29,6 +30,7 @@ import ( type baseBuilder struct { name string pickerBuilder PickerBuilder + config Config } func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { @@ -43,6 +45,7 @@ func (bb *baseBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) // ErrNoSubConnAvailable, because when state of a SubConn changes, we // may call UpdateBalancerState with this picker. picker: NewErrPicker(balancer.ErrNoSubConnAvailable), + config: bb.config, } } @@ -60,6 +63,7 @@ type baseBalancer struct { subConns map[resolver.Address]balancer.SubConn scStates map[balancer.SubConn]connectivity.State picker balancer.Picker + config Config } func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { @@ -74,7 +78,7 @@ func (b *baseBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) addrsSet[a] = struct{}{} if _, ok := b.subConns[a]; !ok { // a is a new address (not existing in b.subConns). - sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{}) + sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: b.config.HealthCheck}) if err != nil { grpclog.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) continue diff --git a/vendor/google.golang.org/grpc/balancer/base/base.go b/vendor/google.golang.org/grpc/balancer/base/base.go index 012ace2f2..34b1f2994 100644 --- a/vendor/google.golang.org/grpc/balancer/base/base.go +++ b/vendor/google.golang.org/grpc/balancer/base/base.go @@ -45,8 +45,20 @@ type PickerBuilder interface { // NewBalancerBuilder returns a balancer builder. The balancers // built by this builder will use the picker builder to build pickers. func NewBalancerBuilder(name string, pb PickerBuilder) balancer.Builder { + return NewBalancerBuilderWithConfig(name, pb, Config{}) +} + +// Config contains the config info about the base balancer builder. +type Config struct { + // HealthCheck indicates whether health checking should be enabled for this specific balancer. + HealthCheck bool +} + +// NewBalancerBuilderWithConfig returns a base balancer builder configured by the provided config. +func NewBalancerBuilderWithConfig(name string, pb PickerBuilder, config Config) balancer.Builder { return &baseBuilder{ name: name, pickerBuilder: pb, + config: config, } } diff --git a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go index 2eda0a1c2..57aea9fb4 100644 --- a/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go +++ b/vendor/google.golang.org/grpc/balancer/roundrobin/roundrobin.go @@ -22,9 +22,9 @@ package roundrobin import ( + "context" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/grpclog" @@ -36,7 +36,7 @@ const Name = "round_robin" // newBuilder creates a new roundrobin balancer builder. func newBuilder() balancer.Builder { - return base.NewBalancerBuilder(Name, &rrPickerBuilder{}) + return base.NewBalancerBuilderWithConfig(Name, &rrPickerBuilder{}, base.Config{HealthCheck: true}) } func init() { diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go index 1ab95fde2..77b684775 100644 --- a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -229,8 +229,13 @@ func (ccb *ccBalancerWrapper) UpdateBalancerState(s connectivity.State, p balanc if ccb.subConns == nil { return } - ccb.cc.csMgr.updateState(s) + // Update picker before updating state. Even though the ordering here does + // not matter, it can lead to multiple calls of Pick in the common start-up + // case where we wait for ready and then perform an RPC. If the picker is + // updated later, we could call the "connecting" picker when the state is + // updated, and then call the "ready" picker after the picker gets updated. ccb.cc.blockingpicker.updatePicker(p) + ccb.cc.csMgr.updateState(s) } func (ccb *ccBalancerWrapper) ResolveNow(o resolver.ResolveNowOption) { diff --git a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go index e0ce32cfb..ca07c154b 100644 --- a/vendor/google.golang.org/grpc/balancer_v1_wrapper.go +++ b/vendor/google.golang.org/grpc/balancer_v1_wrapper.go @@ -19,10 +19,10 @@ package grpc import ( + "context" "strings" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" diff --git a/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go b/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go new file mode 100644 index 000000000..f393bb661 --- /dev/null +++ b/vendor/google.golang.org/grpc/binarylog/grpc_binarylog_v1/binarylog.pb.go @@ -0,0 +1,900 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: grpc/binarylog/grpc_binarylog_v1/binarylog.proto + +package grpc_binarylog_v1 // import "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" +import duration "github.com/golang/protobuf/ptypes/duration" +import timestamp "github.com/golang/protobuf/ptypes/timestamp" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +// Enumerates the type of event +// Note the terminology is different from the RPC semantics +// definition, but the same meaning is expressed here. +type GrpcLogEntry_EventType int32 + +const ( + GrpcLogEntry_EVENT_TYPE_UNKNOWN GrpcLogEntry_EventType = 0 + // Header sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER GrpcLogEntry_EventType = 1 + // Header sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_HEADER GrpcLogEntry_EventType = 2 + // Message sent from client to server + GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE GrpcLogEntry_EventType = 3 + // Message sent from server to client + GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE GrpcLogEntry_EventType = 4 + // A signal that client is done sending + GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE GrpcLogEntry_EventType = 5 + // Trailer indicates the end of the RPC. + // On client side, this event means a trailer was either received + // from the network or the gRPC library locally generated a status + // to inform the application about a failure. + // On server side, this event means the server application requested + // to send a trailer. Note: EVENT_TYPE_CANCEL may still arrive after + // this due to races on server side. + GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER GrpcLogEntry_EventType = 6 + // A signal that the RPC is cancelled. On client side, this + // indicates the client application requests a cancellation. + // On server side, this indicates that cancellation was detected. + // Note: This marks the end of the RPC. Events may arrive after + // this due to races. For example, on client side a trailer + // may arrive even though the application requested to cancel the RPC. + GrpcLogEntry_EVENT_TYPE_CANCEL GrpcLogEntry_EventType = 7 +) + +var GrpcLogEntry_EventType_name = map[int32]string{ + 0: "EVENT_TYPE_UNKNOWN", + 1: "EVENT_TYPE_CLIENT_HEADER", + 2: "EVENT_TYPE_SERVER_HEADER", + 3: "EVENT_TYPE_CLIENT_MESSAGE", + 4: "EVENT_TYPE_SERVER_MESSAGE", + 5: "EVENT_TYPE_CLIENT_HALF_CLOSE", + 6: "EVENT_TYPE_SERVER_TRAILER", + 7: "EVENT_TYPE_CANCEL", +} +var GrpcLogEntry_EventType_value = map[string]int32{ + "EVENT_TYPE_UNKNOWN": 0, + "EVENT_TYPE_CLIENT_HEADER": 1, + "EVENT_TYPE_SERVER_HEADER": 2, + "EVENT_TYPE_CLIENT_MESSAGE": 3, + "EVENT_TYPE_SERVER_MESSAGE": 4, + "EVENT_TYPE_CLIENT_HALF_CLOSE": 5, + "EVENT_TYPE_SERVER_TRAILER": 6, + "EVENT_TYPE_CANCEL": 7, +} + +func (x GrpcLogEntry_EventType) String() string { + return proto.EnumName(GrpcLogEntry_EventType_name, int32(x)) +} +func (GrpcLogEntry_EventType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 0} +} + +// Enumerates the entity that generates the log entry +type GrpcLogEntry_Logger int32 + +const ( + GrpcLogEntry_LOGGER_UNKNOWN GrpcLogEntry_Logger = 0 + GrpcLogEntry_LOGGER_CLIENT GrpcLogEntry_Logger = 1 + GrpcLogEntry_LOGGER_SERVER GrpcLogEntry_Logger = 2 +) + +var GrpcLogEntry_Logger_name = map[int32]string{ + 0: "LOGGER_UNKNOWN", + 1: "LOGGER_CLIENT", + 2: "LOGGER_SERVER", +} +var GrpcLogEntry_Logger_value = map[string]int32{ + "LOGGER_UNKNOWN": 0, + "LOGGER_CLIENT": 1, + "LOGGER_SERVER": 2, +} + +func (x GrpcLogEntry_Logger) String() string { + return proto.EnumName(GrpcLogEntry_Logger_name, int32(x)) +} +func (GrpcLogEntry_Logger) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0, 1} +} + +type Address_Type int32 + +const ( + Address_TYPE_UNKNOWN Address_Type = 0 + // address is in 1.2.3.4 form + Address_TYPE_IPV4 Address_Type = 1 + // address is in IPv6 canonical form (RFC5952 section 4) + // The scope is NOT included in the address string. + Address_TYPE_IPV6 Address_Type = 2 + // address is UDS string + Address_TYPE_UNIX Address_Type = 3 +) + +var Address_Type_name = map[int32]string{ + 0: "TYPE_UNKNOWN", + 1: "TYPE_IPV4", + 2: "TYPE_IPV6", + 3: "TYPE_UNIX", +} +var Address_Type_value = map[string]int32{ + "TYPE_UNKNOWN": 0, + "TYPE_IPV4": 1, + "TYPE_IPV6": 2, + "TYPE_UNIX": 3, +} + +func (x Address_Type) String() string { + return proto.EnumName(Address_Type_name, int32(x)) +} +func (Address_Type) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{7, 0} +} + +// Log entry we store in binary logs +type GrpcLogEntry struct { + // The timestamp of the binary log message + Timestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // Uniquely identifies a call. The value must not be 0 in order to disambiguate + // from an unset value. + // Each call may have several log entries, they will all have the same call_id. + // Nothing is guaranteed about their value other than they are unique across + // different RPCs in the same gRPC process. + CallId uint64 `protobuf:"varint,2,opt,name=call_id,json=callId,proto3" json:"call_id,omitempty"` + // The entry sequence id for this call. The first GrpcLogEntry has a + // value of 1, to disambiguate from an unset value. The purpose of + // this field is to detect missing entries in environments where + // durability or ordering is not guaranteed. + SequenceIdWithinCall uint64 `protobuf:"varint,3,opt,name=sequence_id_within_call,json=sequenceIdWithinCall,proto3" json:"sequence_id_within_call,omitempty"` + Type GrpcLogEntry_EventType `protobuf:"varint,4,opt,name=type,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_EventType" json:"type,omitempty"` + Logger GrpcLogEntry_Logger `protobuf:"varint,5,opt,name=logger,proto3,enum=grpc.binarylog.v1.GrpcLogEntry_Logger" json:"logger,omitempty"` + // The logger uses one of the following fields to record the payload, + // according to the type of the log entry. + // + // Types that are valid to be assigned to Payload: + // *GrpcLogEntry_ClientHeader + // *GrpcLogEntry_ServerHeader + // *GrpcLogEntry_Message + // *GrpcLogEntry_Trailer + Payload isGrpcLogEntry_Payload `protobuf_oneof:"payload"` + // true if payload does not represent the full message or metadata. + PayloadTruncated bool `protobuf:"varint,10,opt,name=payload_truncated,json=payloadTruncated,proto3" json:"payload_truncated,omitempty"` + // Peer address information, will only be recorded on the first + // incoming event. On client side, peer is logged on + // EVENT_TYPE_SERVER_HEADER normally or EVENT_TYPE_SERVER_TRAILER in + // the case of trailers-only. On server side, peer is always + // logged on EVENT_TYPE_CLIENT_HEADER. + Peer *Address `protobuf:"bytes,11,opt,name=peer,proto3" json:"peer,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GrpcLogEntry) Reset() { *m = GrpcLogEntry{} } +func (m *GrpcLogEntry) String() string { return proto.CompactTextString(m) } +func (*GrpcLogEntry) ProtoMessage() {} +func (*GrpcLogEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{0} +} +func (m *GrpcLogEntry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GrpcLogEntry.Unmarshal(m, b) +} +func (m *GrpcLogEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GrpcLogEntry.Marshal(b, m, deterministic) +} +func (dst *GrpcLogEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_GrpcLogEntry.Merge(dst, src) +} +func (m *GrpcLogEntry) XXX_Size() int { + return xxx_messageInfo_GrpcLogEntry.Size(m) +} +func (m *GrpcLogEntry) XXX_DiscardUnknown() { + xxx_messageInfo_GrpcLogEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_GrpcLogEntry proto.InternalMessageInfo + +func (m *GrpcLogEntry) GetTimestamp() *timestamp.Timestamp { + if m != nil { + return m.Timestamp + } + return nil +} + +func (m *GrpcLogEntry) GetCallId() uint64 { + if m != nil { + return m.CallId + } + return 0 +} + +func (m *GrpcLogEntry) GetSequenceIdWithinCall() uint64 { + if m != nil { + return m.SequenceIdWithinCall + } + return 0 +} + +func (m *GrpcLogEntry) GetType() GrpcLogEntry_EventType { + if m != nil { + return m.Type + } + return GrpcLogEntry_EVENT_TYPE_UNKNOWN +} + +func (m *GrpcLogEntry) GetLogger() GrpcLogEntry_Logger { + if m != nil { + return m.Logger + } + return GrpcLogEntry_LOGGER_UNKNOWN +} + +type isGrpcLogEntry_Payload interface { + isGrpcLogEntry_Payload() +} + +type GrpcLogEntry_ClientHeader struct { + ClientHeader *ClientHeader `protobuf:"bytes,6,opt,name=client_header,json=clientHeader,proto3,oneof"` +} + +type GrpcLogEntry_ServerHeader struct { + ServerHeader *ServerHeader `protobuf:"bytes,7,opt,name=server_header,json=serverHeader,proto3,oneof"` +} + +type GrpcLogEntry_Message struct { + Message *Message `protobuf:"bytes,8,opt,name=message,proto3,oneof"` +} + +type GrpcLogEntry_Trailer struct { + Trailer *Trailer `protobuf:"bytes,9,opt,name=trailer,proto3,oneof"` +} + +func (*GrpcLogEntry_ClientHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_ServerHeader) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Message) isGrpcLogEntry_Payload() {} + +func (*GrpcLogEntry_Trailer) isGrpcLogEntry_Payload() {} + +func (m *GrpcLogEntry) GetPayload() isGrpcLogEntry_Payload { + if m != nil { + return m.Payload + } + return nil +} + +func (m *GrpcLogEntry) GetClientHeader() *ClientHeader { + if x, ok := m.GetPayload().(*GrpcLogEntry_ClientHeader); ok { + return x.ClientHeader + } + return nil +} + +func (m *GrpcLogEntry) GetServerHeader() *ServerHeader { + if x, ok := m.GetPayload().(*GrpcLogEntry_ServerHeader); ok { + return x.ServerHeader + } + return nil +} + +func (m *GrpcLogEntry) GetMessage() *Message { + if x, ok := m.GetPayload().(*GrpcLogEntry_Message); ok { + return x.Message + } + return nil +} + +func (m *GrpcLogEntry) GetTrailer() *Trailer { + if x, ok := m.GetPayload().(*GrpcLogEntry_Trailer); ok { + return x.Trailer + } + return nil +} + +func (m *GrpcLogEntry) GetPayloadTruncated() bool { + if m != nil { + return m.PayloadTruncated + } + return false +} + +func (m *GrpcLogEntry) GetPeer() *Address { + if m != nil { + return m.Peer + } + return nil +} + +// XXX_OneofFuncs is for the internal use of the proto package. +func (*GrpcLogEntry) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) { + return _GrpcLogEntry_OneofMarshaler, _GrpcLogEntry_OneofUnmarshaler, _GrpcLogEntry_OneofSizer, []interface{}{ + (*GrpcLogEntry_ClientHeader)(nil), + (*GrpcLogEntry_ServerHeader)(nil), + (*GrpcLogEntry_Message)(nil), + (*GrpcLogEntry_Trailer)(nil), + } +} + +func _GrpcLogEntry_OneofMarshaler(msg proto.Message, b *proto.Buffer) error { + m := msg.(*GrpcLogEntry) + // payload + switch x := m.Payload.(type) { + case *GrpcLogEntry_ClientHeader: + b.EncodeVarint(6<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ClientHeader); err != nil { + return err + } + case *GrpcLogEntry_ServerHeader: + b.EncodeVarint(7<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.ServerHeader); err != nil { + return err + } + case *GrpcLogEntry_Message: + b.EncodeVarint(8<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Message); err != nil { + return err + } + case *GrpcLogEntry_Trailer: + b.EncodeVarint(9<<3 | proto.WireBytes) + if err := b.EncodeMessage(x.Trailer); err != nil { + return err + } + case nil: + default: + return fmt.Errorf("GrpcLogEntry.Payload has unexpected type %T", x) + } + return nil +} + +func _GrpcLogEntry_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) { + m := msg.(*GrpcLogEntry) + switch tag { + case 6: // payload.client_header + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ClientHeader) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_ClientHeader{msg} + return true, err + case 7: // payload.server_header + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(ServerHeader) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_ServerHeader{msg} + return true, err + case 8: // payload.message + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Message) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_Message{msg} + return true, err + case 9: // payload.trailer + if wire != proto.WireBytes { + return true, proto.ErrInternalBadWireType + } + msg := new(Trailer) + err := b.DecodeMessage(msg) + m.Payload = &GrpcLogEntry_Trailer{msg} + return true, err + default: + return false, nil + } +} + +func _GrpcLogEntry_OneofSizer(msg proto.Message) (n int) { + m := msg.(*GrpcLogEntry) + // payload + switch x := m.Payload.(type) { + case *GrpcLogEntry_ClientHeader: + s := proto.Size(x.ClientHeader) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_ServerHeader: + s := proto.Size(x.ServerHeader) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_Message: + s := proto.Size(x.Message) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case *GrpcLogEntry_Trailer: + s := proto.Size(x.Trailer) + n += 1 // tag and wire + n += proto.SizeVarint(uint64(s)) + n += s + case nil: + default: + panic(fmt.Sprintf("proto: unexpected type %T in oneof", x)) + } + return n +} + +type ClientHeader struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The name of the RPC method, which looks something like: + // // + // Note the leading "/" character. + MethodName string `protobuf:"bytes,2,opt,name=method_name,json=methodName,proto3" json:"method_name,omitempty"` + // A single process may be used to run multiple virtual + // servers with different identities. + // The authority is the name of such a server identitiy. + // It is typically a portion of the URI in the form of + // or : . + Authority string `protobuf:"bytes,3,opt,name=authority,proto3" json:"authority,omitempty"` + // the RPC timeout + Timeout *duration.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ClientHeader) Reset() { *m = ClientHeader{} } +func (m *ClientHeader) String() string { return proto.CompactTextString(m) } +func (*ClientHeader) ProtoMessage() {} +func (*ClientHeader) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{1} +} +func (m *ClientHeader) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ClientHeader.Unmarshal(m, b) +} +func (m *ClientHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ClientHeader.Marshal(b, m, deterministic) +} +func (dst *ClientHeader) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClientHeader.Merge(dst, src) +} +func (m *ClientHeader) XXX_Size() int { + return xxx_messageInfo_ClientHeader.Size(m) +} +func (m *ClientHeader) XXX_DiscardUnknown() { + xxx_messageInfo_ClientHeader.DiscardUnknown(m) +} + +var xxx_messageInfo_ClientHeader proto.InternalMessageInfo + +func (m *ClientHeader) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *ClientHeader) GetMethodName() string { + if m != nil { + return m.MethodName + } + return "" +} + +func (m *ClientHeader) GetAuthority() string { + if m != nil { + return m.Authority + } + return "" +} + +func (m *ClientHeader) GetTimeout() *duration.Duration { + if m != nil { + return m.Timeout + } + return nil +} + +type ServerHeader struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServerHeader) Reset() { *m = ServerHeader{} } +func (m *ServerHeader) String() string { return proto.CompactTextString(m) } +func (*ServerHeader) ProtoMessage() {} +func (*ServerHeader) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{2} +} +func (m *ServerHeader) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerHeader.Unmarshal(m, b) +} +func (m *ServerHeader) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerHeader.Marshal(b, m, deterministic) +} +func (dst *ServerHeader) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerHeader.Merge(dst, src) +} +func (m *ServerHeader) XXX_Size() int { + return xxx_messageInfo_ServerHeader.Size(m) +} +func (m *ServerHeader) XXX_DiscardUnknown() { + xxx_messageInfo_ServerHeader.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerHeader proto.InternalMessageInfo + +func (m *ServerHeader) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +type Trailer struct { + // This contains only the metadata from the application. + Metadata *Metadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + // The gRPC status code. + StatusCode uint32 `protobuf:"varint,2,opt,name=status_code,json=statusCode,proto3" json:"status_code,omitempty"` + // An original status message before any transport specific + // encoding. + StatusMessage string `protobuf:"bytes,3,opt,name=status_message,json=statusMessage,proto3" json:"status_message,omitempty"` + // The value of the 'grpc-status-details-bin' metadata key. If + // present, this is always an encoded 'google.rpc.Status' message. + StatusDetails []byte `protobuf:"bytes,4,opt,name=status_details,json=statusDetails,proto3" json:"status_details,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Trailer) Reset() { *m = Trailer{} } +func (m *Trailer) String() string { return proto.CompactTextString(m) } +func (*Trailer) ProtoMessage() {} +func (*Trailer) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{3} +} +func (m *Trailer) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Trailer.Unmarshal(m, b) +} +func (m *Trailer) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Trailer.Marshal(b, m, deterministic) +} +func (dst *Trailer) XXX_Merge(src proto.Message) { + xxx_messageInfo_Trailer.Merge(dst, src) +} +func (m *Trailer) XXX_Size() int { + return xxx_messageInfo_Trailer.Size(m) +} +func (m *Trailer) XXX_DiscardUnknown() { + xxx_messageInfo_Trailer.DiscardUnknown(m) +} + +var xxx_messageInfo_Trailer proto.InternalMessageInfo + +func (m *Trailer) GetMetadata() *Metadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *Trailer) GetStatusCode() uint32 { + if m != nil { + return m.StatusCode + } + return 0 +} + +func (m *Trailer) GetStatusMessage() string { + if m != nil { + return m.StatusMessage + } + return "" +} + +func (m *Trailer) GetStatusDetails() []byte { + if m != nil { + return m.StatusDetails + } + return nil +} + +// Message payload, used by CLIENT_MESSAGE and SERVER_MESSAGE +type Message struct { + // Length of the message. It may not be the same as the length of the + // data field, as the logging payload can be truncated or omitted. + Length uint32 `protobuf:"varint,1,opt,name=length,proto3" json:"length,omitempty"` + // May be truncated or omitted. + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{4} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Message.Unmarshal(m, b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) +} +func (dst *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(dst, src) +} +func (m *Message) XXX_Size() int { + return xxx_messageInfo_Message.Size(m) +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetLength() uint32 { + if m != nil { + return m.Length + } + return 0 +} + +func (m *Message) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +// A list of metadata pairs, used in the payload of client header, +// server header, and server trailer. +// Implementations may omit some entries to honor the header limits +// of GRPC_BINARY_LOG_CONFIG. +// +// Header keys added by gRPC are omitted. To be more specific, +// implementations will not log the following entries, and this is +// not to be treated as a truncation: +// - entries handled by grpc that are not user visible, such as those +// that begin with 'grpc-' (with exception of grpc-trace-bin) +// or keys like 'lb-token' +// - transport specific entries, including but not limited to: +// ':path', ':authority', 'content-encoding', 'user-agent', 'te', etc +// - entries added for call credentials +// +// Implementations must always log grpc-trace-bin if it is present. +// Practically speaking it will only be visible on server side because +// grpc-trace-bin is managed by low level client side mechanisms +// inaccessible from the application level. On server side, the +// header is just a normal metadata key. +// The pair will not count towards the size limit. +type Metadata struct { + Entry []*MetadataEntry `protobuf:"bytes,1,rep,name=entry,proto3" json:"entry,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Metadata) Reset() { *m = Metadata{} } +func (m *Metadata) String() string { return proto.CompactTextString(m) } +func (*Metadata) ProtoMessage() {} +func (*Metadata) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{5} +} +func (m *Metadata) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Metadata.Unmarshal(m, b) +} +func (m *Metadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Metadata.Marshal(b, m, deterministic) +} +func (dst *Metadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_Metadata.Merge(dst, src) +} +func (m *Metadata) XXX_Size() int { + return xxx_messageInfo_Metadata.Size(m) +} +func (m *Metadata) XXX_DiscardUnknown() { + xxx_messageInfo_Metadata.DiscardUnknown(m) +} + +var xxx_messageInfo_Metadata proto.InternalMessageInfo + +func (m *Metadata) GetEntry() []*MetadataEntry { + if m != nil { + return m.Entry + } + return nil +} + +// A metadata key value pair +type MetadataEntry struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *MetadataEntry) Reset() { *m = MetadataEntry{} } +func (m *MetadataEntry) String() string { return proto.CompactTextString(m) } +func (*MetadataEntry) ProtoMessage() {} +func (*MetadataEntry) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{6} +} +func (m *MetadataEntry) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_MetadataEntry.Unmarshal(m, b) +} +func (m *MetadataEntry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_MetadataEntry.Marshal(b, m, deterministic) +} +func (dst *MetadataEntry) XXX_Merge(src proto.Message) { + xxx_messageInfo_MetadataEntry.Merge(dst, src) +} +func (m *MetadataEntry) XXX_Size() int { + return xxx_messageInfo_MetadataEntry.Size(m) +} +func (m *MetadataEntry) XXX_DiscardUnknown() { + xxx_messageInfo_MetadataEntry.DiscardUnknown(m) +} + +var xxx_messageInfo_MetadataEntry proto.InternalMessageInfo + +func (m *MetadataEntry) GetKey() string { + if m != nil { + return m.Key + } + return "" +} + +func (m *MetadataEntry) GetValue() []byte { + if m != nil { + return m.Value + } + return nil +} + +// Address information +type Address struct { + Type Address_Type `protobuf:"varint,1,opt,name=type,proto3,enum=grpc.binarylog.v1.Address_Type" json:"type,omitempty"` + Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"` + // only for TYPE_IPV4 and TYPE_IPV6 + IpPort uint32 `protobuf:"varint,3,opt,name=ip_port,json=ipPort,proto3" json:"ip_port,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Address) Reset() { *m = Address{} } +func (m *Address) String() string { return proto.CompactTextString(m) } +func (*Address) ProtoMessage() {} +func (*Address) Descriptor() ([]byte, []int) { + return fileDescriptor_binarylog_264c8c9c551ce911, []int{7} +} +func (m *Address) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Address.Unmarshal(m, b) +} +func (m *Address) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Address.Marshal(b, m, deterministic) +} +func (dst *Address) XXX_Merge(src proto.Message) { + xxx_messageInfo_Address.Merge(dst, src) +} +func (m *Address) XXX_Size() int { + return xxx_messageInfo_Address.Size(m) +} +func (m *Address) XXX_DiscardUnknown() { + xxx_messageInfo_Address.DiscardUnknown(m) +} + +var xxx_messageInfo_Address proto.InternalMessageInfo + +func (m *Address) GetType() Address_Type { + if m != nil { + return m.Type + } + return Address_TYPE_UNKNOWN +} + +func (m *Address) GetAddress() string { + if m != nil { + return m.Address + } + return "" +} + +func (m *Address) GetIpPort() uint32 { + if m != nil { + return m.IpPort + } + return 0 +} + +func init() { + proto.RegisterType((*GrpcLogEntry)(nil), "grpc.binarylog.v1.GrpcLogEntry") + proto.RegisterType((*ClientHeader)(nil), "grpc.binarylog.v1.ClientHeader") + proto.RegisterType((*ServerHeader)(nil), "grpc.binarylog.v1.ServerHeader") + proto.RegisterType((*Trailer)(nil), "grpc.binarylog.v1.Trailer") + proto.RegisterType((*Message)(nil), "grpc.binarylog.v1.Message") + proto.RegisterType((*Metadata)(nil), "grpc.binarylog.v1.Metadata") + proto.RegisterType((*MetadataEntry)(nil), "grpc.binarylog.v1.MetadataEntry") + proto.RegisterType((*Address)(nil), "grpc.binarylog.v1.Address") + proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_EventType", GrpcLogEntry_EventType_name, GrpcLogEntry_EventType_value) + proto.RegisterEnum("grpc.binarylog.v1.GrpcLogEntry_Logger", GrpcLogEntry_Logger_name, GrpcLogEntry_Logger_value) + proto.RegisterEnum("grpc.binarylog.v1.Address_Type", Address_Type_name, Address_Type_value) +} + +func init() { + proto.RegisterFile("grpc/binarylog/grpc_binarylog_v1/binarylog.proto", fileDescriptor_binarylog_264c8c9c551ce911) +} + +var fileDescriptor_binarylog_264c8c9c551ce911 = []byte{ + // 900 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0x51, 0x6f, 0xe3, 0x44, + 0x10, 0x3e, 0x37, 0x69, 0xdc, 0x4c, 0x92, 0xca, 0x5d, 0x95, 0x3b, 0x5f, 0x29, 0x34, 0xb2, 0x04, + 0x0a, 0x42, 0x72, 0xb9, 0x94, 0xeb, 0xf1, 0x02, 0x52, 0x92, 0xfa, 0xd2, 0x88, 0x5c, 0x1a, 0x6d, + 0x72, 0x3d, 0x40, 0x48, 0xd6, 0x36, 0x5e, 0x1c, 0x0b, 0xc7, 0x6b, 0xd6, 0x9b, 0xa0, 0xfc, 0x2c, + 0xde, 0x90, 0xee, 0x77, 0xf1, 0x8e, 0xbc, 0x6b, 0x27, 0xa6, 0x69, 0x0f, 0x09, 0xde, 0x3c, 0xdf, + 0x7c, 0xf3, 0xcd, 0xee, 0x78, 0x66, 0x16, 0xbe, 0xf2, 0x79, 0x3c, 0x3b, 0xbf, 0x0b, 0x22, 0xc2, + 0xd7, 0x21, 0xf3, 0xcf, 0x53, 0xd3, 0xdd, 0x98, 0xee, 0xea, 0xc5, 0xd6, 0x67, 0xc7, 0x9c, 0x09, + 0x86, 0x8e, 0x52, 0x8a, 0xbd, 0x45, 0x57, 0x2f, 0x4e, 0x3e, 0xf5, 0x19, 0xf3, 0x43, 0x7a, 0x2e, + 0x09, 0x77, 0xcb, 0x5f, 0xce, 0xbd, 0x25, 0x27, 0x22, 0x60, 0x91, 0x0a, 0x39, 0x39, 0xbb, 0xef, + 0x17, 0xc1, 0x82, 0x26, 0x82, 0x2c, 0x62, 0x45, 0xb0, 0xde, 0xeb, 0x50, 0xef, 0xf3, 0x78, 0x36, + 0x64, 0xbe, 0x13, 0x09, 0xbe, 0x46, 0xdf, 0x40, 0x75, 0xc3, 0x31, 0xb5, 0xa6, 0xd6, 0xaa, 0xb5, + 0x4f, 0x6c, 0xa5, 0x62, 0xe7, 0x2a, 0xf6, 0x34, 0x67, 0xe0, 0x2d, 0x19, 0x3d, 0x03, 0x7d, 0x46, + 0xc2, 0xd0, 0x0d, 0x3c, 0x73, 0xaf, 0xa9, 0xb5, 0xca, 0xb8, 0x92, 0x9a, 0x03, 0x0f, 0xbd, 0x84, + 0x67, 0x09, 0xfd, 0x6d, 0x49, 0xa3, 0x19, 0x75, 0x03, 0xcf, 0xfd, 0x3d, 0x10, 0xf3, 0x20, 0x72, + 0x53, 0xa7, 0x59, 0x92, 0xc4, 0xe3, 0xdc, 0x3d, 0xf0, 0xde, 0x49, 0x67, 0x8f, 0x84, 0x21, 0xfa, + 0x16, 0xca, 0x62, 0x1d, 0x53, 0xb3, 0xdc, 0xd4, 0x5a, 0x87, 0xed, 0x2f, 0xec, 0x9d, 0xdb, 0xdb, + 0xc5, 0x83, 0xdb, 0xce, 0x8a, 0x46, 0x62, 0xba, 0x8e, 0x29, 0x96, 0x61, 0xe8, 0x3b, 0xa8, 0x84, + 0xcc, 0xf7, 0x29, 0x37, 0xf7, 0xa5, 0xc0, 0xe7, 0xff, 0x26, 0x30, 0x94, 0x6c, 0x9c, 0x45, 0xa1, + 0xd7, 0xd0, 0x98, 0x85, 0x01, 0x8d, 0x84, 0x3b, 0xa7, 0xc4, 0xa3, 0xdc, 0xac, 0xc8, 0x62, 0x9c, + 0x3d, 0x20, 0xd3, 0x93, 0xbc, 0x6b, 0x49, 0xbb, 0x7e, 0x82, 0xeb, 0xb3, 0x82, 0x9d, 0xea, 0x24, + 0x94, 0xaf, 0x28, 0xcf, 0x75, 0xf4, 0x47, 0x75, 0x26, 0x92, 0xb7, 0xd5, 0x49, 0x0a, 0x36, 0xba, + 0x04, 0x7d, 0x41, 0x93, 0x84, 0xf8, 0xd4, 0x3c, 0xc8, 0x7f, 0xcb, 0x8e, 0xc2, 0x1b, 0xc5, 0xb8, + 0x7e, 0x82, 0x73, 0x72, 0x1a, 0x27, 0x38, 0x09, 0x42, 0xca, 0xcd, 0xea, 0xa3, 0x71, 0x53, 0xc5, + 0x48, 0xe3, 0x32, 0x32, 0xfa, 0x12, 0x8e, 0x62, 0xb2, 0x0e, 0x19, 0xf1, 0x5c, 0xc1, 0x97, 0xd1, + 0x8c, 0x08, 0xea, 0x99, 0xd0, 0xd4, 0x5a, 0x07, 0xd8, 0xc8, 0x1c, 0xd3, 0x1c, 0x47, 0x36, 0x94, + 0x63, 0x4a, 0xb9, 0x59, 0x7b, 0x34, 0x43, 0xc7, 0xf3, 0x38, 0x4d, 0x12, 0x2c, 0x79, 0xd6, 0x5f, + 0x1a, 0x54, 0x37, 0x3f, 0x0c, 0x3d, 0x05, 0xe4, 0xdc, 0x3a, 0xa3, 0xa9, 0x3b, 0xfd, 0x71, 0xec, + 0xb8, 0x6f, 0x47, 0xdf, 0x8f, 0x6e, 0xde, 0x8d, 0x8c, 0x27, 0xe8, 0x14, 0xcc, 0x02, 0xde, 0x1b, + 0x0e, 0xd2, 0xef, 0x6b, 0xa7, 0x73, 0xe5, 0x60, 0x43, 0xbb, 0xe7, 0x9d, 0x38, 0xf8, 0xd6, 0xc1, + 0xb9, 0x77, 0x0f, 0x7d, 0x02, 0xcf, 0x77, 0x63, 0xdf, 0x38, 0x93, 0x49, 0xa7, 0xef, 0x18, 0xa5, + 0x7b, 0xee, 0x2c, 0x38, 0x77, 0x97, 0x51, 0x13, 0x4e, 0x1f, 0xc8, 0xdc, 0x19, 0xbe, 0x76, 0x7b, + 0xc3, 0x9b, 0x89, 0x63, 0xec, 0x3f, 0x2c, 0x30, 0xc5, 0x9d, 0xc1, 0xd0, 0xc1, 0x46, 0x05, 0x7d, + 0x04, 0x47, 0x45, 0x81, 0xce, 0xa8, 0xe7, 0x0c, 0x0d, 0xdd, 0xea, 0x42, 0x45, 0xb5, 0x19, 0x42, + 0x70, 0x38, 0xbc, 0xe9, 0xf7, 0x1d, 0x5c, 0xb8, 0xef, 0x11, 0x34, 0x32, 0x4c, 0x65, 0x34, 0xb4, + 0x02, 0xa4, 0x52, 0x18, 0x7b, 0xdd, 0x2a, 0xe8, 0x59, 0xfd, 0xad, 0xf7, 0x1a, 0xd4, 0x8b, 0xcd, + 0x87, 0x5e, 0xc1, 0xc1, 0x82, 0x0a, 0xe2, 0x11, 0x41, 0xb2, 0xe1, 0xfd, 0xf8, 0xc1, 0x2e, 0x51, + 0x14, 0xbc, 0x21, 0xa3, 0x33, 0xa8, 0x2d, 0xa8, 0x98, 0x33, 0xcf, 0x8d, 0xc8, 0x82, 0xca, 0x01, + 0xae, 0x62, 0x50, 0xd0, 0x88, 0x2c, 0x28, 0x3a, 0x85, 0x2a, 0x59, 0x8a, 0x39, 0xe3, 0x81, 0x58, + 0xcb, 0xb1, 0xad, 0xe2, 0x2d, 0x80, 0x2e, 0x40, 0x4f, 0x17, 0x01, 0x5b, 0x0a, 0x39, 0xae, 0xb5, + 0xf6, 0xf3, 0x9d, 0x9d, 0x71, 0x95, 0x6d, 0x26, 0x9c, 0x33, 0xad, 0x3e, 0xd4, 0x8b, 0x1d, 0xff, + 0x9f, 0x0f, 0x6f, 0xfd, 0xa1, 0x81, 0x9e, 0x75, 0xf0, 0xff, 0xaa, 0x40, 0x22, 0x88, 0x58, 0x26, + 0xee, 0x8c, 0x79, 0xaa, 0x02, 0x0d, 0x0c, 0x0a, 0xea, 0x31, 0x8f, 0xa2, 0xcf, 0xe0, 0x30, 0x23, + 0xe4, 0x73, 0xa8, 0xca, 0xd0, 0x50, 0x68, 0x36, 0x7a, 0x05, 0x9a, 0x47, 0x05, 0x09, 0xc2, 0x44, + 0x56, 0xa4, 0x9e, 0xd3, 0xae, 0x14, 0x68, 0xbd, 0x04, 0x3d, 0x8f, 0x78, 0x0a, 0x95, 0x90, 0x46, + 0xbe, 0x98, 0xcb, 0x03, 0x37, 0x70, 0x66, 0x21, 0x04, 0x65, 0x79, 0x8d, 0x3d, 0x19, 0x2f, 0xbf, + 0xad, 0x2e, 0x1c, 0xe4, 0x67, 0x47, 0x97, 0xb0, 0x4f, 0xd3, 0xcd, 0x65, 0x6a, 0xcd, 0x52, 0xab, + 0xd6, 0x6e, 0x7e, 0xe0, 0x9e, 0x72, 0xc3, 0x61, 0x45, 0xb7, 0x5e, 0x41, 0xe3, 0x1f, 0x38, 0x32, + 0xa0, 0xf4, 0x2b, 0x5d, 0xcb, 0xec, 0x55, 0x9c, 0x7e, 0xa2, 0x63, 0xd8, 0x5f, 0x91, 0x70, 0x49, + 0xb3, 0xdc, 0xca, 0xb0, 0xfe, 0xd4, 0x40, 0xcf, 0xe6, 0x18, 0x5d, 0x64, 0xdb, 0x59, 0x93, 0xcb, + 0xf5, 0xec, 0xf1, 0x89, 0xb7, 0x0b, 0x3b, 0xd9, 0x04, 0x9d, 0x28, 0x34, 0xeb, 0xb0, 0xdc, 0x4c, + 0x1f, 0x8f, 0x20, 0x76, 0x63, 0xc6, 0x85, 0xac, 0x6a, 0x03, 0x57, 0x82, 0x78, 0xcc, 0xb8, 0xb0, + 0x1c, 0x28, 0xcb, 0x1d, 0x61, 0x40, 0xfd, 0xde, 0x76, 0x68, 0x40, 0x55, 0x22, 0x83, 0xf1, 0xed, + 0xd7, 0x86, 0x56, 0x34, 0x2f, 0x8d, 0xbd, 0x8d, 0xf9, 0x76, 0x34, 0xf8, 0xc1, 0x28, 0x75, 0x7f, + 0x86, 0xe3, 0x80, 0xed, 0x1e, 0xb2, 0x7b, 0xd8, 0x95, 0xd6, 0x90, 0xf9, 0xe3, 0xb4, 0x51, 0xc7, + 0xda, 0x4f, 0xed, 0xac, 0x71, 0x7d, 0x16, 0x92, 0xc8, 0xb7, 0x19, 0x57, 0x4f, 0xf3, 0x87, 0x5e, + 0xea, 0xbb, 0x8a, 0xec, 0xf2, 0x8b, 0xbf, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe7, 0xf6, 0x4b, 0x50, + 0xd4, 0x07, 0x00, 0x00, +} diff --git a/vendor/google.golang.org/grpc/call.go b/vendor/google.golang.org/grpc/call.go index 180d79d06..100f05dc7 100644 --- a/vendor/google.golang.org/grpc/call.go +++ b/vendor/google.golang.org/grpc/call.go @@ -19,7 +19,7 @@ package grpc import ( - "golang.org/x/net/context" + "context" ) // Invoke sends the RPC request on the wire and returns after response is diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index f49ac3f9b..84b6dbe3e 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -19,6 +19,7 @@ package grpc import ( + "context" "errors" "fmt" "math" @@ -29,15 +30,17 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" _ "google.golang.org/grpc/balancer/roundrobin" // To register roundrobin. "google.golang.org/grpc/codes" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/envconfig" + "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" @@ -123,12 +126,13 @@ func Dial(target string, opts ...DialOption) (*ClientConn, error) { // e.g. to use dns resolver, a "dns:///" prefix should be applied to the target. func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *ClientConn, err error) { cc := &ClientConn{ - target: target, - csMgr: &connectivityStateManager{}, - conns: make(map[*addrConn]struct{}), - dopts: defaultDialOptions(), - blockingpicker: newPickerWrapper(), - czData: new(channelzData), + target: target, + csMgr: &connectivityStateManager{}, + conns: make(map[*addrConn]struct{}), + dopts: defaultDialOptions(), + blockingpicker: newPickerWrapper(), + czData: new(channelzData), + firstResolveEvent: grpcsync.NewEvent(), } cc.retryThrottler.Store((*retryThrottler)(nil)) cc.ctx, cc.cancel = context.WithCancel(context.Background()) @@ -182,7 +186,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * cc.dopts.copts.Dialer = newProxyDialer( func(ctx context.Context, addr string) (net.Conn, error) { network, addr := parseDialTarget(addr) - return dialContext(ctx, network, addr) + return (&net.Dialer{}).DialContext(ctx, network, addr) }, ) } @@ -285,19 +289,14 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } // Build the resolver. - cc.resolverWrapper, err = newCCResolverWrapper(cc) + rWrapper, err := newCCResolverWrapper(cc) if err != nil { return nil, fmt.Errorf("failed to build resolver: %v", err) } - // Start the resolver wrapper goroutine after resolverWrapper is created. - // - // If the goroutine is started before resolverWrapper is ready, the - // following may happen: The goroutine sends updates to cc. cc forwards - // those to balancer. Balancer creates new addrConn. addrConn fails to - // connect, and calls resolveNow(). resolveNow() tries to use the non-ready - // resolverWrapper. - cc.resolverWrapper.start() + cc.mu.Lock() + cc.resolverWrapper = rWrapper + cc.mu.Unlock() // A blocking dial blocks until the clientConn is ready. if cc.dopts.block { for { @@ -306,7 +305,9 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * break } else if cc.dopts.copts.FailOnNonTempDialError && s == connectivity.TransientFailure { if err = cc.blockingpicker.connectionError(); err != nil { - terr, ok := err.(interface{ Temporary() bool }) + terr, ok := err.(interface { + Temporary() bool + }) if ok && !terr.Temporary() { return nil, err } @@ -384,13 +385,13 @@ type ClientConn struct { csMgr *connectivityStateManager balancerBuildOpts balancer.BuildOptions - resolverWrapper *ccResolverWrapper blockingpicker *pickerWrapper - mu sync.RWMutex - sc ServiceConfig - scRaw string - conns map[*addrConn]struct{} + mu sync.RWMutex + resolverWrapper *ccResolverWrapper + sc ServiceConfig + scRaw string + conns map[*addrConn]struct{} // Keepalive parameter can be updated if a GoAway is received. mkp keepalive.ClientParameters curBalancerName string @@ -399,6 +400,8 @@ type ClientConn struct { balancerWrapper *ccBalancerWrapper retryThrottler atomic.Value + firstResolveEvent *grpcsync.Event + channelzID int64 // channelz unique identification number czData *channelzData } @@ -444,6 +447,25 @@ func (cc *ClientConn) scWatcher() { } } +// waitForResolvedAddrs blocks until the resolver has provided addresses or the +// context expires. Returns nil unless the context expires first; otherwise +// returns a status error based on the context. +func (cc *ClientConn) waitForResolvedAddrs(ctx context.Context) error { + // This is on the RPC path, so we use a fast path to avoid the + // more-expensive "select" below after the resolver has returned once. + if cc.firstResolveEvent.HasFired() { + return nil + } + select { + case <-cc.firstResolveEvent.Done(): + return nil + case <-ctx.Done(): + return status.FromContextError(ctx.Err()).Err() + case <-cc.ctx.Done(): + return ErrClientConnClosing + } +} + func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { cc.mu.Lock() defer cc.mu.Unlock() @@ -457,6 +479,7 @@ func (cc *ClientConn) handleResolvedAddrs(addrs []resolver.Address, err error) { } cc.curAddresses = addrs + cc.firstResolveEvent.Fire() if cc.dopts.balancerBuilder == nil { // Only look at balancer types and switch balancer if balancer dial @@ -715,6 +738,12 @@ func (cc *ClientConn) GetMethodConfig(method string) MethodConfig { return m } +func (cc *ClientConn) healthCheckConfig() *healthCheckConfig { + cc.mu.RLock() + defer cc.mu.RUnlock() + return cc.sc.healthCheckConfig +} + func (cc *ClientConn) getTransport(ctx context.Context, failfast bool, method string) (transport.ClientTransport, func(balancer.DoneInfo), error) { hdr, _ := metadata.FromOutgoingContext(ctx) t, done, err := cc.blockingpicker.pick(ctx, failfast, balancer.PickOptions{ @@ -877,6 +906,10 @@ type addrConn struct { acbw balancer.SubConn scopts balancer.NewSubConnOptions + // transport is set when there's a viable transport (note: ac state may not be READY as LB channel + // health checking may require server to report healthy to set ac to READY), and is reset + // to nil when the current transport should no longer be used to create a stream (e.g. after GoAway + // is received, transport is closed, ac has been torn down). transport transport.ClientTransport // The current transport. mu sync.Mutex @@ -903,6 +936,8 @@ type addrConn struct { czData *channelzData successfulHandshake bool + + healthCheckEnabled bool } // Note: this requires a lock on ac.mu. @@ -956,6 +991,8 @@ func (ac *addrConn) resetTransport(resolveNow bool) { return } + // The transport that was used before is no longer viable. + ac.transport = nil // If the connection is READY, a failure must have occurred. // Otherwise, we'll consider this is a transient failure when: // We've exhausted all addresses @@ -1044,7 +1081,10 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts var serverPrefaceReceived bool var clientPrefaceWrote bool + hcCtx, hcCancel := context.WithCancel(ac.ctx) + onGoAway := func(r transport.GoAwayReason) { + hcCancel() ac.mu.Lock() ac.adjustParams(r) ac.mu.Unlock() @@ -1059,6 +1099,7 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts prefaceTimer := time.NewTimer(connectDeadline.Sub(time.Now())) onClose := func() { + hcCancel() close(onCloseCalled) prefaceTimer.Stop() @@ -1087,18 +1128,14 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts serverPrefaceReceived = true if clientPrefaceWrote { ac.successfulHandshake = true - ac.backoffDeadline = time.Time{} - ac.connectDeadline = time.Time{} - ac.addrIdx = 0 - ac.backoffIdx = 0 } prefaceMu.Unlock() ac.mu.Unlock() } - // Do not cancel in the success path because of this issue in Go1.6: https://github.com/golang/go/issues/15078. connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) + defer cancel() if channelz.IsOn() { copts.ChannelzParentID = ac.channelzID } @@ -1108,12 +1145,12 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts if err == nil { prefaceMu.Lock() clientPrefaceWrote = true - if serverPrefaceReceived { + if serverPrefaceReceived || ac.dopts.reqHandshake == envconfig.RequireHandshakeOff { ac.successfulHandshake = true } prefaceMu.Unlock() - if ac.dopts.waitForHandshake { + if ac.dopts.reqHandshake == envconfig.RequireHandshakeOn { select { case <-prefaceTimer.C: // We didn't get the preface in time. @@ -1126,7 +1163,7 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts close(allowedToReset) return nil } - } else { + } else if ac.dopts.reqHandshake == envconfig.RequireHandshakeHybrid { go func() { select { case <-prefaceTimer.C: @@ -1143,7 +1180,6 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts if err != nil { // newTr is either nil, or closed. - cancel() ac.cc.blockingpicker.updateConnectionError(err) ac.mu.Lock() if ac.state == connectivity.Shutdown { @@ -1166,22 +1202,46 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts return err } + // Now there is a viable transport to be use, so set ac.transport to reflect the new viable transport. + ac.mu.Lock() + if ac.state == connectivity.Shutdown { + ac.mu.Unlock() + close(skipReset) + newTr.Close() + return nil + } + ac.transport = newTr + ac.mu.Unlock() + + healthCheckConfig := ac.cc.healthCheckConfig() + // LB channel health checking is only enabled when all the four requirements below are met: + // 1. it is not disabled by the user with the WithDisableHealthCheck DialOption, + // 2. the internal.HealthCheckFunc is set by importing the grpc/healthcheck package, + // 3. a service config with non-empty healthCheckConfig field is provided, + // 4. the current load balancer allows it. + if !ac.cc.dopts.disableHealthCheck && healthCheckConfig != nil && ac.scopts.HealthCheckEnabled { + if internal.HealthCheckFunc != nil { + go ac.startHealthCheck(hcCtx, newTr, addr, healthCheckConfig.ServiceName) + close(allowedToReset) + return nil + } + // TODO: add a link to the health check doc in the error message. + grpclog.Error("the client side LB channel health check function has not been set.") + } + + // No LB channel health check case ac.mu.Lock() if ac.state == connectivity.Shutdown { ac.mu.Unlock() - // We don't want to reset during this close because we prefer to kick out of this function and let the loop - // in resetTransport take care of reconnecting. + // unblock onGoAway/onClose callback. close(skipReset) - - newTr.Close() return errConnClosing } ac.updateConnectivityState(connectivity.Ready) ac.cc.handleSubConnStateChange(ac.acbw, ac.state) - ac.transport = newTr ac.curAddr = addr ac.mu.Unlock() @@ -1192,6 +1252,51 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts return nil } +func (ac *addrConn) startHealthCheck(ctx context.Context, newTr transport.ClientTransport, addr resolver.Address, serviceName string) { + // Set up the health check helper functions + newStream := func() (interface{}, error) { + return ac.newClientStream(ctx, &StreamDesc{ServerStreams: true}, "/grpc.health.v1.Health/Watch", newTr) + } + firstReady := true + reportHealth := func(ok bool) { + ac.mu.Lock() + defer ac.mu.Unlock() + if ac.transport != newTr { + return + } + if ok { + if firstReady { + firstReady = false + ac.curAddr = addr + } + if ac.state != connectivity.Ready { + ac.updateConnectivityState(connectivity.Ready) + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + } + } else { + if ac.state != connectivity.TransientFailure { + ac.updateConnectivityState(connectivity.TransientFailure) + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + } + } + } + + err := internal.HealthCheckFunc(ctx, newStream, reportHealth, serviceName) + if err != nil { + if status.Code(err) == codes.Unimplemented { + if channelz.IsOn() { + channelz.AddTraceEvent(ac.channelzID, &channelz.TraceEventDesc{ + Desc: "Subchannel health check is unimplemented at server side, thus health check is disabled", + Severity: channelz.CtError, + }) + } + grpclog.Error("Subchannel health check is unimplemented at server side, thus health check is disabled") + } else { + grpclog.Errorf("HealthCheckFunc exits with unexpected error %v", err) + } + } +} + // nextAddr increments the addrIdx if there are more addresses to try. If // there are no more addrs to try it will re-resolve, set addrIdx to 0, and // increment the backoffIdx. @@ -1200,11 +1305,14 @@ func (ac *addrConn) createTransport(backoffNum int, addr resolver.Address, copts func (ac *addrConn) nextAddr() error { ac.mu.Lock() - // If a handshake has been observed, we expect the counters to have manually - // been reset so we'll just return, since we want the next usage to start - // at index 0. + // If a handshake has been observed, we want the next usage to start at + // index 0 immediately. if ac.successfulHandshake { ac.successfulHandshake = false + ac.backoffDeadline = time.Time{} + ac.connectDeadline = time.Time{} + ac.addrIdx = 0 + ac.backoffIdx = 0 ac.mu.Unlock() return nil } @@ -1279,6 +1387,8 @@ func (ac *addrConn) tearDown(err error) { ac.mu.Unlock() return } + curTr := ac.transport + ac.transport = nil // We have to set the state to Shutdown before anything else to prevent races // between setting the state and logic that waits on context cancelation / etc. ac.updateConnectivityState(connectivity.Shutdown) @@ -1286,14 +1396,14 @@ func (ac *addrConn) tearDown(err error) { ac.tearDownErr = err ac.cc.handleSubConnStateChange(ac.acbw, ac.state) ac.curAddr = resolver.Address{} - if err == errConnDrain && ac.transport != nil { + if err == errConnDrain && curTr != nil { // GracefulClose(...) may be executed multiple times when // i) receiving multiple GoAway frames from the server; or // ii) there are concurrent name resolver/Balancer triggered // address removal and GoAway. // We have to unlock and re-lock here because GracefulClose => Close => onClose, which requires locking ac.mu. ac.mu.Unlock() - ac.transport.GracefulClose() + curTr.GracefulClose() ac.mu.Lock() } if channelz.IsOn() { diff --git a/vendor/google.golang.org/grpc/connectivity/connectivity.go b/vendor/google.golang.org/grpc/connectivity/connectivity.go index 568ef5dc6..b1d7dbc54 100644 --- a/vendor/google.golang.org/grpc/connectivity/connectivity.go +++ b/vendor/google.golang.org/grpc/connectivity/connectivity.go @@ -22,7 +22,8 @@ package connectivity import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/grpclog" ) diff --git a/vendor/google.golang.org/grpc/credentials/credentials.go b/vendor/google.golang.org/grpc/credentials/credentials.go index 6c2b811fd..a85156045 100644 --- a/vendor/google.golang.org/grpc/credentials/credentials.go +++ b/vendor/google.golang.org/grpc/credentials/credentials.go @@ -23,6 +23,7 @@ package credentials // import "google.golang.org/grpc/credentials" import ( + "context" "crypto/tls" "crypto/x509" "errors" @@ -32,7 +33,7 @@ import ( "strings" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" + "google.golang.org/grpc/credentials/internal" ) // alpnProtoStr are the specified application level protocols for gRPC. @@ -138,8 +139,8 @@ func (t TLSInfo) AuthType() string { return "tls" } -// GetChannelzSecurityValue returns security info requested by channelz. -func (t TLSInfo) GetChannelzSecurityValue() ChannelzSecurityValue { +// GetSecurityValue returns security info requested by channelz. +func (t TLSInfo) GetSecurityValue() ChannelzSecurityValue { v := &TLSChannelzSecurityValue{ StandardName: cipherSuiteLookup[t.State.CipherSuite], } @@ -187,7 +188,7 @@ func (c *tlsCreds) ClientHandshake(ctx context.Context, authority string, rawCon case <-ctx.Done(): return nil, nil, ctx.Err() } - return tlsConn{Conn: conn, rawConn: rawConn}, TLSInfo{conn.ConnectionState()}, nil + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil } func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) { @@ -195,7 +196,7 @@ func (c *tlsCreds) ServerHandshake(rawConn net.Conn) (net.Conn, AuthInfo, error) if err := conn.Handshake(); err != nil { return nil, nil, err } - return tlsConn{Conn: conn, rawConn: rawConn}, TLSInfo{conn.ConnectionState()}, nil + return internal.WrapSyscallConn(rawConn, conn), TLSInfo{conn.ConnectionState()}, nil } func (c *tlsCreds) Clone() TransportCredentials { @@ -285,11 +286,6 @@ type OtherChannelzSecurityValue struct { func (*OtherChannelzSecurityValue) isChannelzSecurityValue() {} -type tlsConn struct { - *tls.Conn - rawConn net.Conn -} - var cipherSuiteLookup = map[uint16]string{ tls.TLS_RSA_WITH_RC4_128_SHA: "TLS_RSA_WITH_RC4_128_SHA", tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA: "TLS_RSA_WITH_3DES_EDE_CBC_SHA", @@ -309,4 +305,24 @@ var cipherSuiteLookup = map[uint16]string{ tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", tls.TLS_FALLBACK_SCSV: "TLS_FALLBACK_SCSV", + tls.TLS_RSA_WITH_AES_128_CBC_SHA256: "TLS_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256", + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", +} + +// cloneTLSConfig returns a shallow clone of the exported +// fields of cfg, ignoring the unexported sync.Once, which +// contains a mutex and must not be copied. +// +// If cfg is nil, a new zero tls.Config is returned. +// +// TODO: inline this function if possible. +func cloneTLSConfig(cfg *tls.Config) *tls.Config { + if cfg == nil { + return &tls.Config{} + } + + return cfg.Clone() } diff --git a/vendor/google.golang.org/grpc/credentials/go16.go b/vendor/google.golang.org/grpc/credentials/go16.go deleted file mode 100644 index d6bbcc9fd..000000000 --- a/vendor/google.golang.org/grpc/credentials/go16.go +++ /dev/null @@ -1,57 +0,0 @@ -// +build !go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - } -} diff --git a/vendor/google.golang.org/grpc/credentials/go17.go b/vendor/google.golang.org/grpc/credentials/go17.go deleted file mode 100644 index fbd500002..000000000 --- a/vendor/google.golang.org/grpc/credentials/go17.go +++ /dev/null @@ -1,59 +0,0 @@ -// +build go1.7,!go1.8 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled, - Renegotiation: cfg.Renegotiation, - } -} diff --git a/vendor/google.golang.org/grpc/credentials/go18.go b/vendor/google.golang.org/grpc/credentials/go18.go deleted file mode 100644 index db30d46cc..000000000 --- a/vendor/google.golang.org/grpc/credentials/go18.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package credentials - -import ( - "crypto/tls" -) - -func init() { - cipherSuiteLookup[tls.TLS_RSA_WITH_AES_128_CBC_SHA256] = "TLS_RSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256] = "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256] = "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256" - cipherSuiteLookup[tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305] = "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" - cipherSuiteLookup[tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305] = "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305" -} - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// If cfg is nil, a new zero tls.Config is returned. -func cloneTLSConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} - } - - return cfg.Clone() -} diff --git a/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go b/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go new file mode 100644 index 000000000..2f4472bec --- /dev/null +++ b/vendor/google.golang.org/grpc/credentials/internal/syscallconn.go @@ -0,0 +1,61 @@ +// +build !appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains credentials-internal code. +package internal + +import ( + "net" + "syscall" +) + +type sysConn = syscall.Conn + +// syscallConn keeps reference of rawConn to support syscall.Conn for channelz. +// SyscallConn() (the method in interface syscall.Conn) is explicitly +// implemented on this type, +// +// Interface syscall.Conn is implemented by most net.Conn implementations (e.g. +// TCPConn, UnixConn), but is not part of net.Conn interface. So wrapper conns +// that embed net.Conn don't implement syscall.Conn. (Side note: tls.Conn +// doesn't embed net.Conn, so even if syscall.Conn is part of net.Conn, it won't +// help here). +type syscallConn struct { + net.Conn + // sysConn is a type alias of syscall.Conn. It's necessary because the name + // `Conn` collides with `net.Conn`. + sysConn +} + +// WrapSyscallConn tries to wrap rawConn and newConn into a net.Conn that +// implements syscall.Conn. rawConn will be used to support syscall, and newConn +// will be used for read/write. +// +// This function returns newConn if rawConn doesn't implement syscall.Conn. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + sysConn, ok := rawConn.(syscall.Conn) + if !ok { + return newConn + } + return &syscallConn{ + Conn: newConn, + sysConn: sysConn, + } +} diff --git a/vendor/google.golang.org/grpc/naming/go18.go b/vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go similarity index 73% rename from vendor/google.golang.org/grpc/naming/go18.go rename to vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go index b5a0f8427..d4346e9ea 100644 --- a/vendor/google.golang.org/grpc/naming/go18.go +++ b/vendor/google.golang.org/grpc/credentials/internal/syscallconn_appengine.go @@ -1,8 +1,8 @@ -// +build go1.8 +// +build appengine /* * - * Copyright 2017 gRPC authors. + * Copyright 2018 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ * */ -package naming +package internal -import "net" - -var ( - lookupHost = net.DefaultResolver.LookupHost - lookupSRV = net.DefaultResolver.LookupSRV +import ( + "net" ) + +// WrapSyscallConn returns newConn on appengine. +func WrapSyscallConn(rawConn, newConn net.Conn) net.Conn { + return newConn +} diff --git a/vendor/google.golang.org/grpc/dialoptions.go b/vendor/google.golang.org/grpc/dialoptions.go index 99b495272..fe00a254d 100644 --- a/vendor/google.golang.org/grpc/dialoptions.go +++ b/vendor/google.golang.org/grpc/dialoptions.go @@ -19,11 +19,11 @@ package grpc import ( + "context" "fmt" "net" "time" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal" @@ -55,10 +55,11 @@ type dialOptions struct { balancerBuilder balancer.Builder // This is to support grpclb. resolverBuilder resolver.Builder - waitForHandshake bool + reqHandshake envconfig.RequireHandshakeSetting channelzParentID int64 disableServiceConfig bool disableRetry bool + disableHealthCheck bool } // DialOption configures how we set up the connection. @@ -91,10 +92,15 @@ func newFuncDialOption(f func(*dialOptions)) *funcDialOption { } // WithWaitForHandshake blocks until the initial settings frame is received from -// the server before assigning RPCs to the connection. Experimental API. +// the server before assigning RPCs to the connection. +// +// Deprecated: this will become the default behavior in the 1.17 release, and +// will be removed after the 1.18 release. To override the default behavior in +// the 1.17 release, either use this dial option or set the environment +// variable GRPC_GO_READY_BEFORE_HANDSHAKE=on. func WithWaitForHandshake() DialOption { return newFuncDialOption(func(o *dialOptions) { - o.waitForHandshake = true + o.reqHandshake = envconfig.RequireHandshakeOn }) } @@ -454,9 +460,18 @@ func WithMaxHeaderListSize(s uint32) DialOption { }) } +// WithDisableHealthCheck disables the LB channel health checking for all SubConns of this ClientConn. +// +// This API is EXPERIMENTAL. +func WithDisableHealthCheck() DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.disableHealthCheck = true + }) +} func defaultDialOptions() dialOptions { return dialOptions{ disableRetry: !envconfig.Retry, + reqHandshake: envconfig.RequireHandshake, copts: transport.ConnectOptions{ WriteBufferSize: defaultWriteBufSize, ReadBufferSize: defaultReadBufSize, diff --git a/vendor/google.golang.org/grpc/go16.go b/vendor/google.golang.org/grpc/go16.go deleted file mode 100644 index b1db21af6..000000000 --- a/vendor/google.golang.org/grpc/go16.go +++ /dev/null @@ -1,71 +0,0 @@ -// +build go1.6,!go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "fmt" - "io" - "net" - "net/http" - - "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/status" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) -} - -func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { - req.Cancel = ctx.Done() - if err := req.Write(conn); err != nil { - return fmt.Errorf("failed to write the HTTP request: %v", err) - } - return nil -} - -// toRPCErr converts an error into an error from the status package. -func toRPCErr(err error) error { - if err == nil || err == io.EOF { - return err - } - if err == io.ErrUnexpectedEOF { - return status.Error(codes.Internal, err.Error()) - } - if _, ok := status.FromError(err); ok { - return err - } - switch e := err.(type) { - case transport.ConnectionError: - return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - } - return status.Error(codes.Unknown, err.Error()) -} diff --git a/vendor/google.golang.org/grpc/go17.go b/vendor/google.golang.org/grpc/go17.go deleted file mode 100644 index 71a72e8fe..000000000 --- a/vendor/google.golang.org/grpc/go17.go +++ /dev/null @@ -1,72 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package grpc - -import ( - "context" - "fmt" - "io" - "net" - "net/http" - - netctx "golang.org/x/net/context" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/internal/transport" - "google.golang.org/grpc/status" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, address) -} - -func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { - req = req.WithContext(ctx) - if err := req.Write(conn); err != nil { - return fmt.Errorf("failed to write the HTTP request: %v", err) - } - return nil -} - -// toRPCErr converts an error into an error from the status package. -func toRPCErr(err error) error { - if err == nil || err == io.EOF { - return err - } - if err == io.ErrUnexpectedEOF { - return status.Error(codes.Internal, err.Error()) - } - if _, ok := status.FromError(err); ok { - return err - } - switch e := err.(type) { - case transport.ConnectionError: - return status.Error(codes.Unavailable, e.Desc) - default: - switch err { - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - } - return status.Error(codes.Unknown, err.Error()) -} diff --git a/vendor/google.golang.org/grpc/health/client.go b/vendor/google.golang.org/grpc/health/client.go new file mode 100644 index 000000000..e15f04c22 --- /dev/null +++ b/vendor/google.golang.org/grpc/health/client.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package health + +import ( + "context" + "fmt" + "io" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/internal" + "google.golang.org/grpc/internal/backoff" + "google.golang.org/grpc/status" +) + +const maxDelay = 120 * time.Second + +var backoffStrategy = backoff.Exponential{MaxDelay: maxDelay} +var backoffFunc = func(ctx context.Context, retries int) bool { + d := backoffStrategy.Backoff(retries) + timer := time.NewTimer(d) + select { + case <-timer.C: + return true + case <-ctx.Done(): + timer.Stop() + return false + } +} + +func init() { + internal.HealthCheckFunc = clientHealthCheck +} + +func clientHealthCheck(ctx context.Context, newStream func() (interface{}, error), reportHealth func(bool), service string) error { + tryCnt := 0 + +retryConnection: + for { + // Backs off if the connection has failed in some way without receiving a message in the previous retry. + if tryCnt > 0 && !backoffFunc(ctx, tryCnt-1) { + return nil + } + tryCnt++ + + if ctx.Err() != nil { + return nil + } + rawS, err := newStream() + if err != nil { + continue retryConnection + } + + s, ok := rawS.(grpc.ClientStream) + // Ideally, this should never happen. But if it happens, the server is marked as healthy for LBing purposes. + if !ok { + reportHealth(true) + return fmt.Errorf("newStream returned %v (type %T); want grpc.ClientStream", rawS, rawS) + } + + if err = s.SendMsg(&healthpb.HealthCheckRequest{Service: service}); err != nil && err != io.EOF { + // Stream should have been closed, so we can safely continue to create a new stream. + continue retryConnection + } + s.CloseSend() + + resp := new(healthpb.HealthCheckResponse) + for { + err = s.RecvMsg(resp) + + // Reports healthy for the LBing purposes if health check is not implemented in the server. + if status.Code(err) == codes.Unimplemented { + reportHealth(true) + return err + } + + // Reports unhealthy if server's Watch method gives an error other than UNIMPLEMENTED. + if err != nil { + reportHealth(false) + continue retryConnection + } + + // As a message has been received, removes the need for backoff for the next retry by reseting the try count. + tryCnt = 0 + reportHealth(resp.Status == healthpb.HealthCheckResponse_SERVING) + } + } +} diff --git a/vendor/google.golang.org/grpc/health/health.go b/vendor/google.golang.org/grpc/health/server.go similarity index 81% rename from vendor/google.golang.org/grpc/health/health.go rename to vendor/google.golang.org/grpc/health/server.go index 10666f2d3..c86e49988 100644 --- a/vendor/google.golang.org/grpc/health/health.go +++ b/vendor/google.golang.org/grpc/health/server.go @@ -18,15 +18,16 @@ //go:generate ./regenerate.sh -// Package health provides some utility functions to health-check a server. The implementation -// is based on protobuf. Users need to write their own implementations if other IDLs are used. +// Package health provides a service that exposes server's health and it must be +// imported to enable support for client-side health checks. package health import ( + "context" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/codes" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" healthpb "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/status" ) @@ -36,14 +37,14 @@ type Server struct { mu sync.Mutex // statusMap stores the serving status of the services this Server monitors. statusMap map[string]healthpb.HealthCheckResponse_ServingStatus - updates map[string]map[healthpb.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus + updates map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus } // NewServer returns a new Server. func NewServer() *Server { return &Server{ statusMap: map[string]healthpb.HealthCheckResponse_ServingStatus{"": healthpb.HealthCheckResponse_SERVING}, - updates: make(map[string]map[healthpb.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), + updates: make(map[string]map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus), } } @@ -60,7 +61,7 @@ func (s *Server) Check(ctx context.Context, in *healthpb.HealthCheckRequest) (*h } // Watch implements `service Health`. -func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthpb.Health_WatchServer) error { +func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthgrpc.Health_WatchServer) error { service := in.Service // update channel is used for getting service status updates. update := make(chan healthpb.HealthCheckResponse_ServingStatus, 1) @@ -74,7 +75,7 @@ func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthpb.Health_W // Registers the update channel to the correct place in the updates map. if _, ok := s.updates[service]; !ok { - s.updates[service] = make(map[healthpb.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) + s.updates[service] = make(map[healthgrpc.Health_WatchServer]chan healthpb.HealthCheckResponse_ServingStatus) } s.updates[service][stream] = update defer func() { @@ -83,10 +84,16 @@ func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthpb.Health_W s.mu.Unlock() }() s.mu.Unlock() + + var lastSentStatus healthpb.HealthCheckResponse_ServingStatus = -1 for { select { // Status updated. Sends the up-to-date status to the client. case servingStatus := <-update: + if lastSentStatus == servingStatus { + continue + } + lastSentStatus = servingStatus err := stream.Send(&healthpb.HealthCheckResponse{Status: servingStatus}) if err != nil { return status.Error(codes.Canceled, "Stream has ended.") @@ -102,6 +109,8 @@ func (s *Server) Watch(in *healthpb.HealthCheckRequest, stream healthpb.Health_W // or insert a new service entry into the statusMap. func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthCheckResponse_ServingStatus) { s.mu.Lock() + defer s.mu.Unlock() + s.statusMap[service] = servingStatus for _, update := range s.updates[service] { // Clears previous updates, that are not sent to the client, from the channel. @@ -113,5 +122,4 @@ func (s *Server) SetServingStatus(service string, servingStatus healthpb.HealthC // Puts the most recent update to the channel. update <- servingStatus } - s.mu.Unlock() } diff --git a/vendor/google.golang.org/grpc/interceptor.go b/vendor/google.golang.org/grpc/interceptor.go index 1f6ef6780..8b7350022 100644 --- a/vendor/google.golang.org/grpc/interceptor.go +++ b/vendor/google.golang.org/grpc/interceptor.go @@ -19,7 +19,7 @@ package grpc import ( - "golang.org/x/net/context" + "context" ) // UnaryInvoker is called by UnaryClientInterceptor to complete RPCs. diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go new file mode 100644 index 000000000..fee6aecd0 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package binarylog implementation binary logging as defined in +// https://github.com/grpc/proposal/blob/master/A16-binary-logging.md. +package binarylog + +import ( + "fmt" + "os" + + "google.golang.org/grpc/grpclog" +) + +// Logger is the global binary logger. It can be used to get binary logger for +// each method. +type Logger interface { + getMethodLogger(methodName string) *MethodLogger +} + +// binLogger is the global binary logger for the binary. One of this should be +// built at init time from the configuration (environment varialbe or flags). +// +// It is used to get a methodLogger for each individual method. +var binLogger Logger + +// SetLogger sets the binarg logger. +// +// Only call this at init time. +func SetLogger(l Logger) { + binLogger = l +} + +// GetMethodLogger returns the methodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each methodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func GetMethodLogger(methodName string) *MethodLogger { + if binLogger == nil { + return nil + } + return binLogger.getMethodLogger(methodName) +} + +func init() { + const envStr = "GRPC_BINARY_LOG_FILTER" + configStr := os.Getenv(envStr) + binLogger = NewLoggerFromConfigString(configStr) +} + +type methodLoggerConfig struct { + // Max length of header and message. + hdr, msg uint64 +} + +type logger struct { + all *methodLoggerConfig + services map[string]*methodLoggerConfig + methods map[string]*methodLoggerConfig + + blacklist map[string]struct{} +} + +// newEmptyLogger creates an empty logger. The map fields need to be filled in +// using the set* functions. +func newEmptyLogger() *logger { + return &logger{} +} + +// Set method logger for "*". +func (l *logger) setDefaultMethodLogger(ml *methodLoggerConfig) error { + if l.all != nil { + return fmt.Errorf("conflicting global rules found") + } + l.all = ml + return nil +} + +// Set method logger for "service/*". +// +// New methodLogger with same service overrides the old one. +func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) error { + if _, ok := l.services[service]; ok { + return fmt.Errorf("conflicting rules for service %v found", service) + } + if l.services == nil { + l.services = make(map[string]*methodLoggerConfig) + } + l.services[service] = ml + return nil +} + +// Set method logger for "service/method". +// +// New methodLogger with same method overrides the old one. +func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) error { + if _, ok := l.blacklist[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if _, ok := l.methods[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if l.methods == nil { + l.methods = make(map[string]*methodLoggerConfig) + } + l.methods[method] = ml + return nil +} + +// Set blacklist method for "-service/method". +func (l *logger) setBlacklist(method string) error { + if _, ok := l.blacklist[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if _, ok := l.methods[method]; ok { + return fmt.Errorf("conflicting rules for method %v found", method) + } + if l.blacklist == nil { + l.blacklist = make(map[string]struct{}) + } + l.blacklist[method] = struct{}{} + return nil +} + +// getMethodLogger returns the methodLogger for the given methodName. +// +// methodName should be in the format of "/service/method". +// +// Each methodLogger returned by this method is a new instance. This is to +// generate sequence id within the call. +func (l *logger) getMethodLogger(methodName string) *MethodLogger { + s, m, err := parseMethodName(methodName) + if err != nil { + grpclog.Infof("binarylogging: failed to parse %q: %v", methodName, err) + return nil + } + if ml, ok := l.methods[s+"/"+m]; ok { + return newMethodLogger(ml.hdr, ml.msg) + } + if _, ok := l.blacklist[s+"/"+m]; ok { + return nil + } + if ml, ok := l.services[s]; ok { + return newMethodLogger(ml.hdr, ml.msg) + } + if l.all == nil { + return nil + } + return newMethodLogger(l.all.hdr, l.all.msg) +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go new file mode 100644 index 000000000..1ee00a39a --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog_testutil.go @@ -0,0 +1,42 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// This file contains exported variables/functions that are exported for testing +// only. +// +// An ideal way for this would be to put those in a *_test.go but in binarylog +// package. But this doesn't work with staticcheck with go module. Error was: +// "MdToMetadataProto not declared by package binarylog". This could be caused +// by the way staticcheck looks for files for a certain package, which doesn't +// support *_test.go files. +// +// Move those to binary_test.go when staticcheck is fixed. + +package binarylog + +var ( + // AllLogger is a logger that logs all headers/messages for all RPCs. It's + // for testing only. + AllLogger = NewLoggerFromConfigString("*") + // MdToMetadataProto converts metadata to a binary logging proto message. + // It's for testing only. + MdToMetadataProto = mdToMetadataProto + // AddrToProto converts an address to a binary logging proto message. It's + // for testing only. + AddrToProto = addrToProto +) diff --git a/vendor/google.golang.org/grpc/internal/binarylog/env_config.go b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go new file mode 100644 index 000000000..eb188eae5 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go @@ -0,0 +1,210 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "google.golang.org/grpc/grpclog" +) + +// NewLoggerFromConfigString reads the string and build a logger. It can be used +// to build a new logger and assign it to binarylog.Logger. +// +// Example filter config strings: +// - "" Nothing will be logged +// - "*" All headers and messages will be fully logged. +// - "*{h}" Only headers will be logged. +// - "*{m:256}" Only the first 256 bytes of each message will be logged. +// - "Foo/*" Logs every method in service Foo +// - "Foo/*,-Foo/Bar" Logs every method in service Foo except method /Foo/Bar +// - "Foo/*,Foo/Bar{m:256}" Logs the first 256 bytes of each message in method +// /Foo/Bar, logs all headers and messages in every other method in service +// Foo. +// +// If two configs exist for one certain method or service, the one specified +// later overrides the privous config. +func NewLoggerFromConfigString(s string) Logger { + if s == "" { + return nil + } + l := newEmptyLogger() + methods := strings.Split(s, ",") + for _, method := range methods { + if err := l.fillMethodLoggerWithConfigString(method); err != nil { + grpclog.Warningf("failed to parse binary log config: %v", err) + return nil + } + } + return l +} + +// fillMethodLoggerWithConfigString parses config, creates methodLogger and adds +// it to the right map in the logger. +func (l *logger) fillMethodLoggerWithConfigString(config string) error { + // "" is invalid. + if config == "" { + return errors.New("empty string is not a valid method binary logging config") + } + + // "-service/method", blacklist, no * or {} allowed. + if config[0] == '-' { + s, m, suffix, err := parseMethodConfigAndSuffix(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if m == "*" { + return fmt.Errorf("invalid config: %q, %v", config, "* not allowd in blacklist config") + } + if suffix != "" { + return fmt.Errorf("invalid config: %q, %v", config, "header/message limit not allowed in blacklist config") + } + if err := l.setBlacklist(s + "/" + m); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + // "*{h:256;m:256}" + if config[0] == '*' { + hdr, msg, err := parseHeaderMessageLengthConfig(config[1:]) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + if err := l.setDefaultMethodLogger(&methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + return nil + } + + s, m, suffix, err := parseMethodConfigAndSuffix(config) + if err != nil { + return fmt.Errorf("invalid config: %q, %v", config, err) + } + hdr, msg, err := parseHeaderMessageLengthConfig(suffix) + if err != nil { + return fmt.Errorf("invalid header/message length config: %q, %v", suffix, err) + } + if m == "*" { + if err := l.setServiceMethodLogger(s, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } else { + if err := l.setMethodMethodLogger(s+"/"+m, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + return fmt.Errorf("invalid config: %v", err) + } + } + return nil +} + +const ( + // TODO: this const is only used by env_config now. But could be useful for + // other config. Move to binarylog.go if necessary. + maxUInt = ^uint64(0) + + // For "p.s/m" plus any suffix. Suffix will be parsed again. See test for + // expected output. + longMethodConfigRegexpStr = `^([\w./]+)/((?:\w+)|[*])(.+)?$` + + // For suffix from above, "{h:123,m:123}". See test for expected output. + optionalLengthRegexpStr = `(?::(\d+))?` // Optional ":123". + headerConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `}$` + messageConfigRegexpStr = `^{m` + optionalLengthRegexpStr + `}$` + headerMessageConfigRegexpStr = `^{h` + optionalLengthRegexpStr + `;m` + optionalLengthRegexpStr + `}$` +) + +var ( + longMethodConfigRegexp = regexp.MustCompile(longMethodConfigRegexpStr) + headerConfigRegexp = regexp.MustCompile(headerConfigRegexpStr) + messageConfigRegexp = regexp.MustCompile(messageConfigRegexpStr) + headerMessageConfigRegexp = regexp.MustCompile(headerMessageConfigRegexpStr) +) + +// Turn "service/method{h;m}" into "service", "method", "{h;m}". +func parseMethodConfigAndSuffix(c string) (service, method, suffix string, _ error) { + // Regexp result: + // + // in: "p.s/m{h:123,m:123}", + // out: []string{"p.s/m{h:123,m:123}", "p.s", "m", "{h:123,m:123}"}, + match := longMethodConfigRegexp.FindStringSubmatch(c) + if match == nil { + return "", "", "", fmt.Errorf("%q contains invalid substring", c) + } + service = match[1] + method = match[2] + suffix = match[3] + return +} + +// Turn "{h:123;m:345}" into 123, 345. +// +// Return maxUInt if length is unspecified. +func parseHeaderMessageLengthConfig(c string) (hdrLenStr, msgLenStr uint64, err error) { + if c == "" { + return maxUInt, maxUInt, nil + } + // Header config only. + if match := headerConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to convert %q to uint", s) + } + return hdrLenStr, 0, nil + } + return maxUInt, 0, nil + } + + // Message config only. + if match := messageConfigRegexp.FindStringSubmatch(c); match != nil { + if s := match[1]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("Failed to convert %q to uint", s) + } + return 0, msgLenStr, nil + } + return 0, maxUInt, nil + } + + // Header and message config both. + if match := headerMessageConfigRegexp.FindStringSubmatch(c); match != nil { + // Both hdr and msg are specified, but one or two of them might be empty. + hdrLenStr = maxUInt + msgLenStr = maxUInt + if s := match[1]; s != "" { + hdrLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("Failed to convert %q to uint", s) + } + } + if s := match[2]; s != "" { + msgLenStr, err = strconv.ParseUint(s, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("Failed to convert %q to uint", s) + } + } + return hdrLenStr, msgLenStr, nil + } + return 0, 0, fmt.Errorf("%q contains invalid substring", c) +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go new file mode 100644 index 000000000..b06cdd4d4 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go @@ -0,0 +1,426 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "net" + "strings" + "sync/atomic" + "time" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type callIDGenerator struct { + id uint64 +} + +func (g *callIDGenerator) next() uint64 { + id := atomic.AddUint64(&g.id, 1) + return id +} + +// reset is for testing only, and doesn't need to be thread safe. +func (g *callIDGenerator) reset() { + g.id = 0 +} + +var idGen callIDGenerator + +// MethodLogger is the sub-logger for each method. +type MethodLogger struct { + headerMaxLen, messageMaxLen uint64 + + callID uint64 + idWithinCallGen *callIDGenerator + + sink Sink // TODO(blog): make this plugable. +} + +func newMethodLogger(h, m uint64) *MethodLogger { + return &MethodLogger{ + headerMaxLen: h, + messageMaxLen: m, + + callID: idGen.next(), + idWithinCallGen: &callIDGenerator{}, + + sink: defaultSink, // TODO(blog): make it plugable. + } +} + +// Log creates a proto binary log entry, and logs it to the sink. +func (ml *MethodLogger) Log(c LogEntryConfig) { + m := c.toProto() + timestamp, _ := ptypes.TimestampProto(time.Now()) + m.Timestamp = timestamp + m.CallId = ml.callID + m.SequenceIdWithinCall = ml.idWithinCallGen.next() + + switch pay := m.Payload.(type) { + case *pb.GrpcLogEntry_ClientHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ClientHeader.GetMetadata()) + case *pb.GrpcLogEntry_ServerHeader: + m.PayloadTruncated = ml.truncateMetadata(pay.ServerHeader.GetMetadata()) + case *pb.GrpcLogEntry_Message: + m.PayloadTruncated = ml.truncateMessage(pay.Message) + } + + ml.sink.Write(m) +} + +func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { + if ml.headerMaxLen == maxUInt { + return false + } + var ( + bytesLimit = ml.headerMaxLen + index int + ) + // At the end of the loop, index will be the first entry where the total + // size is greater than the limit: + // + // len(entry[:index]) <= ml.hdr && len(entry[:index+1]) > ml.hdr. + for ; index < len(mdPb.Entry); index++ { + entry := mdPb.Entry[index] + if entry.Key == "grpc-trace-bin" { + // "grpc-trace-bin" is a special key. It's kept in the log entry, + // but not counted towards the size limit. + continue + } + currentEntryLen := uint64(len(entry.Value)) + if currentEntryLen > bytesLimit { + break + } + bytesLimit -= currentEntryLen + } + truncated = index < len(mdPb.Entry) + mdPb.Entry = mdPb.Entry[:index] + return truncated +} + +func (ml *MethodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { + if ml.messageMaxLen == maxUInt { + return false + } + if ml.messageMaxLen >= uint64(len(msgPb.Data)) { + return false + } + msgPb.Data = msgPb.Data[:ml.messageMaxLen] + return true +} + +// LogEntryConfig represents the configuration for binary log entry. +type LogEntryConfig interface { + toProto() *pb.GrpcLogEntry +} + +// ClientHeader configs the binary log entry to be a ClientHeader entry. +type ClientHeader struct { + OnClientSide bool + Header metadata.MD + MethodName string + Authority string + Timeout time.Duration + // PeerAddr is required only when it's on server side. + PeerAddr net.Addr +} + +func (c *ClientHeader) toProto() *pb.GrpcLogEntry { + // This function doesn't need to set all the fields (e.g. seq ID). The Log + // function will set the fields when necessary. + clientHeader := &pb.ClientHeader{ + Metadata: mdToMetadataProto(c.Header), + MethodName: c.MethodName, + Authority: c.Authority, + } + if c.Timeout > 0 { + clientHeader.Timeout = ptypes.DurationProto(c.Timeout) + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HEADER, + Payload: &pb.GrpcLogEntry_ClientHeader{ + ClientHeader: clientHeader, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ServerHeader configs the binary log entry to be a ServerHeader entry. +type ServerHeader struct { + OnClientSide bool + Header metadata.MD + // PeerAddr is required only when it's on client side. + PeerAddr net.Addr +} + +func (c *ServerHeader) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_HEADER, + Payload: &pb.GrpcLogEntry_ServerHeader{ + ServerHeader: &pb.ServerHeader{ + Metadata: mdToMetadataProto(c.Header), + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// ClientMessage configs the binary log entry to be a ClientMessage entry. +type ClientMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message interface{} +} + +func (c *ClientMessage) toProto() *pb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_MESSAGE, + Payload: &pb.GrpcLogEntry_Message{ + Message: &pb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerMessage configs the binary log entry to be a ServerMessage entry. +type ServerMessage struct { + OnClientSide bool + // Message can be a proto.Message or []byte. Other messages formats are not + // supported. + Message interface{} +} + +func (c *ServerMessage) toProto() *pb.GrpcLogEntry { + var ( + data []byte + err error + ) + if m, ok := c.Message.(proto.Message); ok { + data, err = proto.Marshal(m) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal proto message: %v", err) + } + } else if b, ok := c.Message.([]byte); ok { + data = b + } else { + grpclog.Infof("binarylogging: message to log is neither proto.message nor []byte") + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_MESSAGE, + Payload: &pb.GrpcLogEntry_Message{ + Message: &pb.Message{ + Length: uint32(len(data)), + Data: data, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ClientHalfClose configs the binary log entry to be a ClientHalfClose entry. +type ClientHalfClose struct { + OnClientSide bool +} + +func (c *ClientHalfClose) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CLIENT_HALF_CLOSE, + Payload: nil, // No payload here. + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// ServerTrailer configs the binary log entry to be a ServerTrailer entry. +type ServerTrailer struct { + OnClientSide bool + Trailer metadata.MD + // Err is the status error. + Err error + // PeerAddr is required only when it's on client side and the RPC is trailer + // only. + PeerAddr net.Addr +} + +func (c *ServerTrailer) toProto() *pb.GrpcLogEntry { + st, ok := status.FromError(c.Err) + if !ok { + grpclog.Info("binarylogging: error in trailer is not a status error") + } + var ( + detailsBytes []byte + err error + ) + stProto := st.Proto() + if stProto != nil && len(stProto.Details) != 0 { + detailsBytes, err = proto.Marshal(stProto) + if err != nil { + grpclog.Infof("binarylogging: failed to marshal status proto: %v", err) + } + } + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_SERVER_TRAILER, + Payload: &pb.GrpcLogEntry_Trailer{ + Trailer: &pb.Trailer{ + Metadata: mdToMetadataProto(c.Trailer), + StatusCode: uint32(st.Code()), + StatusMessage: st.Message(), + StatusDetails: detailsBytes, + }, + }, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + if c.PeerAddr != nil { + ret.Peer = addrToProto(c.PeerAddr) + } + return ret +} + +// Cancel configs the binary log entry to be a Cancel entry. +type Cancel struct { + OnClientSide bool +} + +func (c *Cancel) toProto() *pb.GrpcLogEntry { + ret := &pb.GrpcLogEntry{ + Type: pb.GrpcLogEntry_EVENT_TYPE_CANCEL, + Payload: nil, + } + if c.OnClientSide { + ret.Logger = pb.GrpcLogEntry_LOGGER_CLIENT + } else { + ret.Logger = pb.GrpcLogEntry_LOGGER_SERVER + } + return ret +} + +// metadataKeyOmit returns whether the metadata entry with this key should be +// omitted. +func metadataKeyOmit(key string) bool { + switch key { + case "lb-token", ":path", ":authority", "content-encoding", "content-type", "user-agent", "te": + return true + case "grpc-trace-bin": // grpc-trace-bin is special because it's visiable to users. + return false + } + if strings.HasPrefix(key, "grpc-") { + return true + } + return false +} + +func mdToMetadataProto(md metadata.MD) *pb.Metadata { + ret := &pb.Metadata{} + for k, vv := range md { + if metadataKeyOmit(k) { + continue + } + for _, v := range vv { + ret.Entry = append(ret.Entry, + &pb.MetadataEntry{ + Key: k, + Value: []byte(v), + }, + ) + } + } + return ret +} + +func addrToProto(addr net.Addr) *pb.Address { + ret := &pb.Address{} + switch a := addr.(type) { + case *net.TCPAddr: + if a.IP.To4() != nil { + ret.Type = pb.Address_TYPE_IPV4 + } else if a.IP.To16() != nil { + ret.Type = pb.Address_TYPE_IPV6 + } else { + ret.Type = pb.Address_TYPE_UNKNOWN + // Do not set address and port fields. + break + } + ret.Address = a.IP.String() + ret.IpPort = uint32(a.Port) + case *net.UnixAddr: + ret.Type = pb.Address_TYPE_UNIX + ret.Address = a.String() + default: + ret.Type = pb.Address_TYPE_UNKNOWN + } + return ret +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/sink.go b/vendor/google.golang.org/grpc/internal/binarylog/sink.go new file mode 100644 index 000000000..20d044f0f --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/binarylog/sink.go @@ -0,0 +1,162 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package binarylog + +import ( + "bufio" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "sync" + "time" + + "github.com/golang/protobuf/proto" + pb "google.golang.org/grpc/binarylog/grpc_binarylog_v1" + "google.golang.org/grpc/grpclog" +) + +var ( + defaultSink Sink = &noopSink{} // TODO(blog): change this default (file in /tmp). +) + +// SetDefaultSink sets the sink where binary logs will be written to. +// +// Not thread safe. Only set during initialization. +func SetDefaultSink(s Sink) { + if defaultSink != nil { + defaultSink.Close() + } + defaultSink = s +} + +// Sink writes log entry into the binary log sink. +type Sink interface { + // Write will be called to write the log entry into the sink. + // + // It should be thread-safe so it can be called in parallel. + Write(*pb.GrpcLogEntry) error + // Close will be called when the Sink is replaced by a new Sink. + Close() error +} + +type noopSink struct{} + +func (ns *noopSink) Write(*pb.GrpcLogEntry) error { return nil } +func (ns *noopSink) Close() error { return nil } + +// newWriterSink creates a binary log sink with the given writer. +// +// Write() marshalls the proto message and writes it to the given writer. Each +// message is prefixed with a 4 byte big endian unsigned integer as the length. +// +// No buffer is done, Close() doesn't try to close the writer. +func newWriterSink(w io.Writer) *writerSink { + return &writerSink{out: w} +} + +type writerSink struct { + out io.Writer +} + +func (ws *writerSink) Write(e *pb.GrpcLogEntry) error { + b, err := proto.Marshal(e) + if err != nil { + grpclog.Infof("binary logging: failed to marshal proto message: %v", err) + } + hdr := make([]byte, 4) + binary.BigEndian.PutUint32(hdr, uint32(len(b))) + if _, err := ws.out.Write(hdr); err != nil { + return err + } + if _, err := ws.out.Write(b); err != nil { + return err + } + return nil +} + +func (ws *writerSink) Close() error { return nil } + +type bufWriteCloserSink struct { + mu sync.Mutex + closer io.Closer + out *writerSink // out is built on buf. + buf *bufio.Writer // buf is kept for flush. + + writeStartOnce sync.Once + writeTicker *time.Ticker +} + +func (fs *bufWriteCloserSink) Write(e *pb.GrpcLogEntry) error { + // Start the write loop when Write is called. + fs.writeStartOnce.Do(fs.startFlushGoroutine) + fs.mu.Lock() + if err := fs.out.Write(e); err != nil { + fs.mu.Unlock() + return err + } + fs.mu.Unlock() + return nil +} + +const ( + bufFlushDuration = 60 * time.Second +) + +func (fs *bufWriteCloserSink) startFlushGoroutine() { + fs.writeTicker = time.NewTicker(bufFlushDuration) + go func() { + for range fs.writeTicker.C { + fs.mu.Lock() + fs.buf.Flush() + fs.mu.Unlock() + } + }() +} + +func (fs *bufWriteCloserSink) Close() error { + if fs.writeTicker != nil { + fs.writeTicker.Stop() + } + fs.mu.Lock() + fs.buf.Flush() + fs.closer.Close() + fs.out.Close() + fs.mu.Unlock() + return nil +} + +func newBufWriteCloserSink(o io.WriteCloser) Sink { + bufW := bufio.NewWriter(o) + return &bufWriteCloserSink{ + closer: o, + out: newWriterSink(bufW), + buf: bufW, + } +} + +// NewTempFileSink creates a temp file and returns a Sink that writes to this +// file. +func NewTempFileSink() (Sink, error) { + tempFile, err := ioutil.TempFile("/tmp", "grpcgo_binarylog_*.txt") + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %v", err) + } + return newBufWriteCloserSink(tempFile), nil +} diff --git a/vendor/google.golang.org/grpc/credentials/go19.go b/vendor/google.golang.org/grpc/internal/binarylog/util.go similarity index 52% rename from vendor/google.golang.org/grpc/credentials/go19.go rename to vendor/google.golang.org/grpc/internal/binarylog/util.go index 2a4ca1a57..15dc7803d 100644 --- a/vendor/google.golang.org/grpc/credentials/go19.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/util.go @@ -1,5 +1,3 @@ -// +build go1.9,!appengine - /* * * Copyright 2018 gRPC authors. @@ -18,18 +16,26 @@ * */ -package credentials +package binarylog import ( "errors" - "syscall" + "strings" ) -// implements the syscall.Conn interface -func (c tlsConn) SyscallConn() (syscall.RawConn, error) { - conn, ok := c.rawConn.(syscall.Conn) - if !ok { - return nil, errors.New("RawConn does not implement syscall.Conn") +// parseMethodName splits service and method from the input. It expects format +// "/service/method". +// +// TODO: move to internal/grpcutil. +func parseMethodName(methodName string) (service, method string, _ error) { + if !strings.HasPrefix(methodName, "/") { + return "", "", errors.New("invalid method name: should start with /") + } + methodName = methodName[1:] + + pos := strings.LastIndex(methodName, "/") + if pos < 0 { + return "", "", errors.New("invalid method name: suffix /method is missing") } - return conn.SyscallConn() + return methodName[:pos], methodName[pos+1:], nil } diff --git a/vendor/google.golang.org/grpc/internal/channelz/funcs.go b/vendor/google.golang.org/grpc/internal/channelz/funcs.go index 6e729fa63..3021a31a5 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/funcs.go +++ b/vendor/google.golang.org/grpc/internal/channelz/funcs.go @@ -558,7 +558,7 @@ func (c *channelMap) GetServerSockets(id int64, startID int64) ([]*SocketMetric, ids = append(ids, k) } sort.Sort(int64Slice(ids)) - idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= id }) + idx := sort.Search(len(ids), func(i int) bool { return ids[i] >= startID }) count := 0 var end bool for i, v := range ids[idx:] { @@ -601,8 +601,11 @@ func (c *channelMap) GetChannel(id int64) *ChannelMetric { } cm.NestedChans = copyMap(cn.nestedChans) cm.SubChans = copyMap(cn.subChans) + // cn.c can be set to &dummyChannel{} when deleteSelfFromMap is called. Save a copy of cn.c when + // holding the lock to prevent potential data race. + chanCopy := cn.c c.mu.RUnlock() - cm.ChannelData = cn.c.ChannelzMetric() + cm.ChannelData = chanCopy.ChannelzMetric() cm.ID = cn.id cm.RefName = cn.refName cm.Trace = cn.trace.dumpData() @@ -620,8 +623,11 @@ func (c *channelMap) GetSubChannel(id int64) *SubChannelMetric { return nil } cm.Sockets = copyMap(sc.sockets) + // sc.c can be set to &dummyChannel{} when deleteSelfFromMap is called. Save a copy of sc.c when + // holding the lock to prevent potential data race. + chanCopy := sc.c c.mu.RUnlock() - cm.ChannelData = sc.c.ChannelzMetric() + cm.ChannelData = chanCopy.ChannelzMetric() cm.ID = sc.id cm.RefName = sc.refName cm.Trace = sc.trace.dumpData() diff --git a/vendor/google.golang.org/grpc/internal/channelz/types_linux.go b/vendor/google.golang.org/grpc/internal/channelz/types_linux.go index 07215396d..692dd6181 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types_linux.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types_linux.go @@ -1,4 +1,4 @@ -// +build !appengine,go1.7 +// +build !appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go b/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go index b24600480..79edbefc4 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types_nonlinux.go @@ -1,4 +1,4 @@ -// +build !linux appengine !go1.7 +// +build !linux appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go b/vendor/google.golang.org/grpc/internal/channelz/util_linux.go similarity index 96% rename from vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go rename to vendor/google.golang.org/grpc/internal/channelz/util_linux.go index e1e9e32d7..fdf409d55 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/util_linux_go19.go +++ b/vendor/google.golang.org/grpc/internal/channelz/util_linux.go @@ -1,4 +1,4 @@ -// +build linux,go1.9,!appengine +// +build linux,!appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go b/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go similarity index 95% rename from vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go rename to vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go index 1d4da952d..8864a0811 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux_pre_go19.go +++ b/vendor/google.golang.org/grpc/internal/channelz/util_nonlinux.go @@ -1,4 +1,4 @@ -// +build !linux !go1.9 appengine +// +build !linux appengine /* * diff --git a/vendor/google.golang.org/grpc/internal/envconfig/envconfig.go b/vendor/google.golang.org/grpc/internal/envconfig/envconfig.go index 3ee8740f1..a3e02b661 100644 --- a/vendor/google.golang.org/grpc/internal/envconfig/envconfig.go +++ b/vendor/google.golang.org/grpc/internal/envconfig/envconfig.go @@ -25,11 +25,45 @@ import ( ) const ( - prefix = "GRPC_GO_" - retryStr = prefix + "RETRY" + prefix = "GRPC_GO_" + retryStr = prefix + "RETRY" + requireHandshakeStr = prefix + "REQUIRE_HANDSHAKE" +) + +// RequireHandshakeSetting describes the settings for handshaking. +type RequireHandshakeSetting int + +const ( + // RequireHandshakeHybrid (default, deprecated) indicates to wait for + // handshake before considering a connection ready, but wait before + // considering successful. + RequireHandshakeHybrid RequireHandshakeSetting = iota + // RequireHandshakeOn (default after the 1.17 release) indicates to wait + // for handshake before considering a connection ready/successful. + RequireHandshakeOn + // RequireHandshakeOff indicates to not wait for handshake before + // considering a connection ready/successful. + RequireHandshakeOff ) var ( // Retry is set if retry is explicitly enabled via "GRPC_GO_RETRY=on". Retry = strings.EqualFold(os.Getenv(retryStr), "on") + // RequireHandshake is set based upon the GRPC_GO_REQUIRE_HANDSHAKE + // environment variable. + // + // Will be removed after the 1.18 release. + RequireHandshake RequireHandshakeSetting ) + +func init() { + switch strings.ToLower(os.Getenv(requireHandshakeStr)) { + case "on": + RequireHandshake = RequireHandshakeOn + case "off": + RequireHandshake = RequireHandshakeOff + case "hybrid": + // Will be removed after the 1.17 release. + RequireHandshake = RequireHandshakeHybrid + } +} diff --git a/vendor/google.golang.org/grpc/internal/grpcsync/event.go b/vendor/google.golang.org/grpc/internal/grpcsync/event.go new file mode 100644 index 000000000..fbe697c37 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/grpcsync/event.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcsync implements additional synchronization primitives built upon +// the sync package. +package grpcsync + +import ( + "sync" + "sync/atomic" +) + +// Event represents a one-time event that may occur in the future. +type Event struct { + fired int32 + c chan struct{} + o sync.Once +} + +// Fire causes e to complete. It is safe to call multiple times, and +// concurrently. It returns true iff this call to Fire caused the signaling +// channel returned by Done to close. +func (e *Event) Fire() bool { + ret := false + e.o.Do(func() { + atomic.StoreInt32(&e.fired, 1) + close(e.c) + ret = true + }) + return ret +} + +// Done returns a channel that will be closed when Fire is called. +func (e *Event) Done() <-chan struct{} { + return e.c +} + +// HasFired returns true if Fire has been called. +func (e *Event) HasFired() bool { + return atomic.LoadInt32(&e.fired) == 1 +} + +// NewEvent returns a new, ready-to-use Event. +func NewEvent() *Event { + return &Event{c: make(chan struct{})} +} diff --git a/vendor/google.golang.org/grpc/internal/internal.go b/vendor/google.golang.org/grpc/internal/internal.go index 41f8af678..f8932b1d8 100644 --- a/vendor/google.golang.org/grpc/internal/internal.go +++ b/vendor/google.golang.org/grpc/internal/internal.go @@ -20,7 +20,7 @@ // symbols to avoid circular dependencies. package internal -import "golang.org/x/net/context" +import "context" var ( // WithContextDialer is exported by clientconn.go diff --git a/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go b/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go new file mode 100644 index 000000000..43281a3e0 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/syscall/syscall_linux.go @@ -0,0 +1,114 @@ +// +build !appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package syscall provides functionalities that grpc uses to get low-level operating system +// stats/info. +package syscall + +import ( + "fmt" + "net" + "syscall" + "time" + + "golang.org/x/sys/unix" + "google.golang.org/grpc/grpclog" +) + +// GetCPUTime returns the how much CPU time has passed since the start of this process. +func GetCPUTime() int64 { + var ts unix.Timespec + if err := unix.ClockGettime(unix.CLOCK_PROCESS_CPUTIME_ID, &ts); err != nil { + grpclog.Fatal(err) + } + return ts.Nano() +} + +// Rusage is an alias for syscall.Rusage under linux non-appengine environment. +type Rusage syscall.Rusage + +// GetRusage returns the resource usage of current process. +func GetRusage() (rusage *Rusage) { + rusage = new(Rusage) + syscall.Getrusage(syscall.RUSAGE_SELF, (*syscall.Rusage)(rusage)) + return +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. +func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { + f := (*syscall.Rusage)(first) + l := (*syscall.Rusage)(latest) + var ( + utimeDiffs = l.Utime.Sec - f.Utime.Sec + utimeDiffus = l.Utime.Usec - f.Utime.Usec + stimeDiffs = l.Stime.Sec - f.Stime.Sec + stimeDiffus = l.Stime.Usec - f.Stime.Usec + ) + + uTimeElapsed := float64(utimeDiffs) + float64(utimeDiffus)*1.0e-6 + sTimeElapsed := float64(stimeDiffs) + float64(stimeDiffus)*1.0e-6 + + return uTimeElapsed, sTimeElapsed +} + +// SetTCPUserTimeout sets the TCP user timeout on a connection's socket +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + // not a TCP connection. exit early + return nil + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + return fmt.Errorf("error getting raw connection: %v", err) + } + err = rawConn.Control(func(fd uintptr) { + err = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, int(timeout/time.Millisecond)) + }) + if err != nil { + return fmt.Errorf("error setting option on socket: %v", err) + } + + return nil +} + +// GetTCPUserTimeout gets the TCP user timeout on a connection's socket +func GetTCPUserTimeout(conn net.Conn) (opt int, err error) { + tcpconn, ok := conn.(*net.TCPConn) + if !ok { + err = fmt.Errorf("conn is not *net.TCPConn. got %T", conn) + return + } + rawConn, err := tcpconn.SyscallConn() + if err != nil { + err = fmt.Errorf("error getting raw connection: %v", err) + return + } + err = rawConn.Control(func(fd uintptr) { + opt, err = syscall.GetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT) + }) + if err != nil { + err = fmt.Errorf("error getting option on socket: %v", err) + return + } + + return +} diff --git a/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go b/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go new file mode 100644 index 000000000..61678feb0 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/syscall/syscall_nonlinux.go @@ -0,0 +1,63 @@ +// +build !linux appengine + +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package syscall + +import ( + "net" + "time" + + "google.golang.org/grpc/grpclog" +) + +func init() { + grpclog.Info("CPU time info is unavailable on non-linux or appengine environment.") +} + +// GetCPUTime returns the how much CPU time has passed since the start of this process. +// It always returns 0 under non-linux or appengine environment. +func GetCPUTime() int64 { + return 0 +} + +// Rusage is an empty struct under non-linux or appengine environment. +type Rusage struct{} + +// GetRusage is a no-op function under non-linux or appengine environment. +func GetRusage() (rusage *Rusage) { + return nil +} + +// CPUTimeDiff returns the differences of user CPU time and system CPU time used +// between two Rusage structs. It a no-op function for non-linux or appengine environment. +func CPUTimeDiff(first *Rusage, latest *Rusage) (float64, float64) { + return 0, 0 +} + +// SetTCPUserTimeout is a no-op function under non-linux or appengine environments +func SetTCPUserTimeout(conn net.Conn, timeout time.Duration) error { + return nil +} + +// GetTCPUserTimeout is a no-op function under non-linux or appengine environments +// a negative return value indicates the operation is not supported +func GetTCPUserTimeout(conn net.Conn) (int, error) { + return -1, nil +} diff --git a/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go b/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go index 63cd2627c..070680edb 100644 --- a/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go +++ b/vendor/google.golang.org/grpc/internal/transport/bdp_estimator.go @@ -24,9 +24,10 @@ import ( ) const ( - // bdpLimit is the maximum value the flow control windows - // will be increased to. - bdpLimit = (1 << 20) * 4 + // bdpLimit is the maximum value the flow control windows will be increased + // to. TCP typically limits this to 4MB, but some systems go up to 16MB. + // Since this is only a limit, it is safe to make it optimistic. + bdpLimit = (1 << 20) * 16 // alpha is a constant factor used to keep a moving average // of RTTs. alpha = 0.9 diff --git a/vendor/google.golang.org/grpc/internal/transport/go16.go b/vendor/google.golang.org/grpc/internal/transport/go16.go deleted file mode 100644 index e0d00115d..000000000 --- a/vendor/google.golang.org/grpc/internal/transport/go16.go +++ /dev/null @@ -1,52 +0,0 @@ -// +build go1.6,!go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package transport - -import ( - "net" - "net/http" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - "golang.org/x/net/context" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address) -} - -// ContextErr converts the error from context package into a status error. -func ContextErr(err error) error { - switch err { - case context.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) -} - -// contextFromRequest returns a background context. -func contextFromRequest(r *http.Request) context.Context { - return context.Background() -} diff --git a/vendor/google.golang.org/grpc/internal/transport/go17.go b/vendor/google.golang.org/grpc/internal/transport/go17.go deleted file mode 100644 index 4d515b00d..000000000 --- a/vendor/google.golang.org/grpc/internal/transport/go17.go +++ /dev/null @@ -1,53 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package transport - -import ( - "context" - "net" - "net/http" - - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - - netctx "golang.org/x/net/context" -) - -// dialContext connects to the address on the named network. -func dialContext(ctx context.Context, network, address string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, address) -} - -// ContextErr converts the error from context package into a status error. -func ContextErr(err error) error { - switch err { - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return status.Error(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return status.Error(codes.Canceled, err.Error()) - } - return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) -} - -// contextFromRequest returns a context from the HTTP Request. -func contextFromRequest(r *http.Request) context.Context { - return r.Context() -} diff --git a/vendor/google.golang.org/grpc/internal/transport/handler_server.go b/vendor/google.golang.org/grpc/internal/transport/handler_server.go index c6fb4b9c1..73b41ea7e 100644 --- a/vendor/google.golang.org/grpc/internal/transport/handler_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/handler_server.go @@ -24,6 +24,7 @@ package transport import ( + "context" "errors" "fmt" "io" @@ -34,7 +35,6 @@ import ( "time" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" "golang.org/x/net/http2" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -307,7 +307,7 @@ func (ht *serverHandlerTransport) WriteHeader(s *Stream, md metadata.MD) error { func (ht *serverHandlerTransport) HandleStreams(startStream func(*Stream), traceCtx func(context.Context, string) context.Context) { // With this transport type there will be exactly 1 stream: this HTTP request. - ctx := contextFromRequest(ht.req) + ctx := ht.req.Context() var cancel context.CancelFunc if ht.timeoutSet { ctx, cancel = context.WithTimeout(ctx, ht.timeout) diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_client.go b/vendor/google.golang.org/grpc/internal/transport/http2_client.go index 0c3c47e2a..39208b146 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_client.go @@ -19,6 +19,8 @@ package transport import ( + "context" + "fmt" "io" "math" "net" @@ -28,13 +30,13 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/peer" @@ -121,7 +123,7 @@ func dial(ctx context.Context, fn func(context.Context, string) (net.Conn, error if fn != nil { return fn(ctx, addr) } - return dialContext(ctx, "tcp", addr) + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) } func isTemporary(err error) bool { @@ -165,6 +167,21 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne conn.Close() } }(conn) + kp := opts.KeepaliveParams + // Validate keepalive parameters. + if kp.Time == 0 { + kp.Time = defaultClientKeepaliveTime + } + if kp.Timeout == 0 { + kp.Timeout = defaultClientKeepaliveTimeout + } + keepaliveEnabled := false + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + keepaliveEnabled = true + } var ( isSecure bool authInfo credentials.AuthInfo @@ -188,14 +205,6 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne } isSecure = true } - kp := opts.KeepaliveParams - // Validate keepalive parameters. - if kp.Time == 0 { - kp.Time = defaultClientKeepaliveTime - } - if kp.Timeout == 0 { - kp.Timeout = defaultClientKeepaliveTimeout - } dynamicWindow := true icwz := int32(initialWindowSize) if opts.InitialConnWindowSize >= defaultWindowSize { @@ -239,6 +248,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne czData: new(channelzData), onGoAway: onGoAway, onClose: onClose, + keepaliveEnabled: keepaliveEnabled, } t.controlBuf = newControlBuffer(t.ctxDone) if opts.InitialWindowSize >= defaultWindowSize { @@ -265,10 +275,9 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne t.statsHandler.HandleConn(t.ctx, connBegin) } if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, "") + t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) } - if t.kp.Time != infinity { - t.keepaliveEnabled = true + if t.keepaliveEnabled { go t.keepalive() } // Start the reader goroutine for incoming message. Each transport has diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_server.go b/vendor/google.golang.org/grpc/internal/transport/http2_server.go index efb7f53ff..df2740398 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_server.go @@ -20,6 +20,7 @@ package transport import ( "bytes" + "context" "errors" "fmt" "io" @@ -31,7 +32,6 @@ import ( "time" "github.com/golang/protobuf/proto" - "golang.org/x/net/context" "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" @@ -237,7 +237,7 @@ func newHTTP2Server(conn net.Conn, config *ServerConfig) (_ ServerTransport, err t.stats.HandleConn(t.ctx, connBegin) } if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, "") + t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) } t.framer.writer.Flush() diff --git a/vendor/google.golang.org/grpc/internal/transport/transport.go b/vendor/google.golang.org/grpc/internal/transport/transport.go index 1be518a62..4d7e89067 100644 --- a/vendor/google.golang.org/grpc/internal/transport/transport.go +++ b/vendor/google.golang.org/grpc/internal/transport/transport.go @@ -22,6 +22,7 @@ package transport import ( + "context" "errors" "fmt" "io" @@ -29,7 +30,6 @@ import ( "sync" "sync/atomic" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/keepalive" @@ -186,8 +186,12 @@ type Stream struct { headerDone uint32 // set when headerChan is closed. Used to avoid closing headerChan multiple times. // hdrMu protects header and trailer metadata on the server-side. - hdrMu sync.Mutex - header metadata.MD // the received header metadata. + hdrMu sync.Mutex + // On client side, header keeps the received header metadata. + // + // On server side, header keeps the header set by SetHeader(). The complete + // header will merged into this after t.WriteHeader() is called. + header metadata.MD trailer metadata.MD // the key-value map of trailer metadata. noHeaders bool // set if the client never received headers (set only after the stream is done). @@ -266,10 +270,19 @@ func (s *Stream) Done() <-chan struct{} { return s.done } -// Header acquires the key-value pairs of header metadata once it -// is available. It blocks until i) the metadata is ready or ii) there is no -// header metadata or iii) the stream is canceled/expired. +// Header returns the header metadata of the stream. +// +// On client side, it acquires the key-value pairs of header metadata once it is +// available. It blocks until i) the metadata is ready or ii) there is no header +// metadata or iii) the stream is canceled/expired. +// +// On server side, it returns the out header after t.WriteHeader is called. func (s *Stream) Header() (metadata.MD, error) { + if s.headerChan == nil && s.header != nil { + // On server side, return the header in stream. It will be the out + // header after t.WriteHeader is called. + return s.header.Copy(), nil + } err := s.waitOnHeader() // Even if the stream is closed, header is returned if available. select { @@ -710,3 +723,14 @@ type channelzData struct { lastMsgSentTime int64 lastMsgRecvTime int64 } + +// ContextErr converts the error from context package into a status error. +func ContextErr(err error) error { + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + return status.Errorf(codes.Internal, "Unexpected error from context packet: %v", err) +} diff --git a/vendor/google.golang.org/grpc/metadata/metadata.go b/vendor/google.golang.org/grpc/metadata/metadata.go index bd2eaf408..cf6d1b947 100644 --- a/vendor/google.golang.org/grpc/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/metadata/metadata.go @@ -22,10 +22,9 @@ package metadata // import "google.golang.org/grpc/metadata" import ( + "context" "fmt" "strings" - - "golang.org/x/net/context" ) // DecodeKeyValue returns k, v, nil. diff --git a/vendor/google.golang.org/grpc/naming/dns_resolver.go b/vendor/google.golang.org/grpc/naming/dns_resolver.go index 0f8a908ea..fd8cd3b5a 100644 --- a/vendor/google.golang.org/grpc/naming/dns_resolver.go +++ b/vendor/google.golang.org/grpc/naming/dns_resolver.go @@ -19,13 +19,13 @@ package naming import ( + "context" "errors" "fmt" "net" "strconv" "time" - "golang.org/x/net/context" "google.golang.org/grpc/grpclog" ) @@ -37,6 +37,9 @@ const ( var ( errMissingAddr = errors.New("missing address") errWatcherClose = errors.New("watcher has been closed") + + lookupHost = net.DefaultResolver.LookupHost + lookupSRV = net.DefaultResolver.LookupSRV ) // NewDNSResolverWithFreq creates a DNS Resolver that can resolve DNS names, and diff --git a/vendor/google.golang.org/grpc/naming/go17.go b/vendor/google.golang.org/grpc/naming/go17.go deleted file mode 100644 index 57b65d7b8..000000000 --- a/vendor/google.golang.org/grpc/naming/go17.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build go1.6,!go1.8 - -/* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package naming - -import ( - "net" - - "golang.org/x/net/context" -) - -var ( - lookupHost = func(ctx context.Context, host string) ([]string, error) { return net.LookupHost(host) } - lookupSRV = func(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { - return net.LookupSRV(service, proto, name) - } -) diff --git a/vendor/google.golang.org/grpc/peer/peer.go b/vendor/google.golang.org/grpc/peer/peer.go index 317b8b9d0..e01d219ff 100644 --- a/vendor/google.golang.org/grpc/peer/peer.go +++ b/vendor/google.golang.org/grpc/peer/peer.go @@ -21,9 +21,9 @@ package peer import ( + "context" "net" - "golang.org/x/net/context" "google.golang.org/grpc/credentials" ) diff --git a/vendor/google.golang.org/grpc/picker_wrapper.go b/vendor/google.golang.org/grpc/picker_wrapper.go index 76cc456aa..14f915d67 100644 --- a/vendor/google.golang.org/grpc/picker_wrapper.go +++ b/vendor/google.golang.org/grpc/picker_wrapper.go @@ -19,10 +19,10 @@ package grpc import ( + "context" "io" "sync" - "golang.org/x/net/context" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" "google.golang.org/grpc/grpclog" diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go index bda4309c0..d1e38aad7 100644 --- a/vendor/google.golang.org/grpc/pickfirst.go +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -19,7 +19,8 @@ package grpc import ( - "golang.org/x/net/context" + "context" + "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/grpclog" diff --git a/vendor/google.golang.org/grpc/proxy.go b/vendor/google.golang.org/grpc/proxy.go index 2d40236e2..f8f69bfb7 100644 --- a/vendor/google.golang.org/grpc/proxy.go +++ b/vendor/google.golang.org/grpc/proxy.go @@ -20,6 +20,8 @@ package grpc import ( "bufio" + "context" + "encoding/base64" "errors" "fmt" "io" @@ -27,10 +29,10 @@ import ( "net/http" "net/http/httputil" "net/url" - - "golang.org/x/net/context" ) +const proxyAuthHeaderKey = "Proxy-Authorization" + var ( // errDisabled indicates that proxy is disabled for the address. errDisabled = errors.New("proxy is disabled for the address") @@ -38,7 +40,7 @@ var ( httpProxyFromEnvironment = http.ProxyFromEnvironment ) -func mapAddress(ctx context.Context, address string) (string, error) { +func mapAddress(ctx context.Context, address string) (*url.URL, error) { req := &http.Request{ URL: &url.URL{ Scheme: "https", @@ -47,12 +49,12 @@ func mapAddress(ctx context.Context, address string) (string, error) { } url, err := httpProxyFromEnvironment(req) if err != nil { - return "", err + return nil, err } if url == nil { - return "", errDisabled + return nil, errDisabled } - return url.Host, nil + return url, nil } // To read a response from a net.Conn, http.ReadResponse() takes a bufio.Reader. @@ -69,18 +71,28 @@ func (c *bufConn) Read(b []byte) (int, error) { return c.r.Read(b) } -func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ net.Conn, err error) { +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, backendAddr string, proxyURL *url.URL) (_ net.Conn, err error) { defer func() { if err != nil { conn.Close() } }() - req := (&http.Request{ + req := &http.Request{ Method: http.MethodConnect, - URL: &url.URL{Host: addr}, + URL: &url.URL{Host: backendAddr}, Header: map[string][]string{"User-Agent": {grpcUA}}, - }) + } + if t := proxyURL.User; t != nil { + u := t.Username() + p, _ := t.Password() + req.Header.Add(proxyAuthHeaderKey, "Basic "+basicAuth(u, p)) + } if err := sendHTTPRequest(ctx, req, conn); err != nil { return nil, fmt.Errorf("failed to write the HTTP request: %v", err) @@ -108,23 +120,33 @@ func doHTTPConnectHandshake(ctx context.Context, conn net.Conn, addr string) (_ // provided dialer, does HTTP CONNECT handshake and returns the connection. func newProxyDialer(dialer func(context.Context, string) (net.Conn, error)) func(context.Context, string) (net.Conn, error) { return func(ctx context.Context, addr string) (conn net.Conn, err error) { - var skipHandshake bool - newAddr, err := mapAddress(ctx, addr) + var newAddr string + proxyURL, err := mapAddress(ctx, addr) if err != nil { if err != errDisabled { return nil, err } - skipHandshake = true newAddr = addr + } else { + newAddr = proxyURL.Host } conn, err = dialer(ctx, newAddr) if err != nil { return } - if !skipHandshake { - conn, err = doHTTPConnectHandshake(ctx, conn, addr) + if proxyURL != nil { + // proxy is disabled if proxyURL is nil. + conn, err = doHTTPConnectHandshake(ctx, conn, addr, proxyURL) } return } } + +func sendHTTPRequest(ctx context.Context, req *http.Request, conn net.Conn) error { + req = req.WithContext(ctx) + if err := req.Write(conn); err != nil { + return fmt.Errorf("failed to write the HTTP request: %v", err) + } + return nil +} diff --git a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go index 4af67422c..f33189fed 100644 --- a/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go +++ b/vendor/google.golang.org/grpc/resolver/dns/dns_resolver.go @@ -21,6 +21,7 @@ package dns import ( + "context" "encoding/json" "errors" "fmt" @@ -31,7 +32,6 @@ import ( "sync" "time" - "golang.org/x/net/context" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/internal/grpcrand" @@ -43,9 +43,10 @@ func init() { } const ( - defaultPort = "443" - defaultFreq = time.Minute * 30 - golang = "GO" + defaultPort = "443" + defaultFreq = time.Minute * 30 + defaultDNSSvrPort = "53" + golang = "GO" // In DNS, service config is encoded in a TXT record via the mechanism // described in RFC-1464 using the attribute name grpc_config. txtAttribute = "grpc_config=" @@ -61,6 +62,31 @@ var ( errEndsWithColon = errors.New("dns resolver: missing port after port-separator colon") ) +var ( + defaultResolver netResolver = net.DefaultResolver +) + +var customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + var dialer net.Dialer + return dialer.DialContext(ctx, network, authority) + } +} + +var customAuthorityResolver = func(authority string) (netResolver, error) { + host, port, err := parseTarget(authority, defaultDNSSvrPort) + if err != nil { + return nil, err + } + + authorityWithPort := net.JoinHostPort(host, port) + + return &net.Resolver{ + PreferGo: true, + Dial: customAuthorityDialler(authorityWithPort), + }, nil +} + // NewBuilder creates a dnsBuilder which is used to factory DNS resolvers. func NewBuilder() resolver.Builder { return &dnsBuilder{minFreq: defaultFreq} diff --git a/vendor/google.golang.org/grpc/resolver/dns/go19.go b/vendor/google.golang.org/grpc/resolver/dns/go19.go deleted file mode 100644 index 9886de275..000000000 --- a/vendor/google.golang.org/grpc/resolver/dns/go19.go +++ /dev/null @@ -1,54 +0,0 @@ -// +build go1.9 - -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package dns - -import ( - "net" - - "golang.org/x/net/context" -) - -var ( - defaultResolver netResolver = net.DefaultResolver -) - -const defaultDNSSvrPort = "53" - -var customAuthorityDialler = func(authority string) func(ctx context.Context, network, address string) (net.Conn, error) { - return func(ctx context.Context, network, address string) (net.Conn, error) { - var dialer net.Dialer - return dialer.DialContext(ctx, network, authority) - } -} - -var customAuthorityResolver = func(authority string) (netResolver, error) { - host, port, err := parseTarget(authority, defaultDNSSvrPort) - if err != nil { - return nil, err - } - - authorityWithPort := net.JoinHostPort(host, port) - - return &net.Resolver{ - PreferGo: true, - Dial: customAuthorityDialler(authorityWithPort), - }, nil -} diff --git a/vendor/google.golang.org/grpc/resolver/dns/pre_go19.go b/vendor/google.golang.org/grpc/resolver/dns/pre_go19.go deleted file mode 100644 index 70428113b..000000000 --- a/vendor/google.golang.org/grpc/resolver/dns/pre_go19.go +++ /dev/null @@ -1,51 +0,0 @@ -// +build go1.6, !go1.9 - -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package dns - -import ( - "fmt" - "net" - - "golang.org/x/net/context" -) - -var ( - defaultResolver netResolver = &preGo19Resolver{} -) - -type preGo19Resolver struct { -} - -func (*preGo19Resolver) LookupHost(ctx context.Context, host string) ([]string, error) { - return net.LookupHost(host) -} - -func (*preGo19Resolver) LookupSRV(ctx context.Context, service, proto, name string) (string, []*net.SRV, error) { - return net.LookupSRV(service, proto, name) -} - -func (*preGo19Resolver) LookupTXT(ctx context.Context, name string) ([]string, error) { - return net.LookupTXT(name) -} - -var customAuthorityResolver = func(authority string) (netResolver, error) { - return nil, fmt.Errorf("Default DNS resolver does not support custom DNS server with go < 1.9") -} diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go index a6c02ac9e..9d7602539 100644 --- a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -93,47 +93,6 @@ func newCCResolverWrapper(cc *ClientConn) (*ccResolverWrapper, error) { return ccr, nil } -func (ccr *ccResolverWrapper) start() { - go ccr.watcher() -} - -// watcher processes address updates and service config updates sequentially. -// Otherwise, we need to resolve possible races between address and service -// config (e.g. they specify different balancer types). -func (ccr *ccResolverWrapper) watcher() { - for { - select { - case <-ccr.done: - return - default: - } - - select { - case addrs := <-ccr.addrCh: - select { - case <-ccr.done: - return - default: - } - grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) - if channelz.IsOn() { - ccr.addChannelzTraceEvent(addrs) - } - ccr.cc.handleResolvedAddrs(addrs, nil) - case sc := <-ccr.scCh: - select { - case <-ccr.done: - return - default: - } - grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) - ccr.cc.handleServiceConfig(sc) - case <-ccr.done: - return - } - } -} - func (ccr *ccResolverWrapper) resolveNow(o resolver.ResolveNowOption) { ccr.resolver.ResolveNow(o) } @@ -146,20 +105,27 @@ func (ccr *ccResolverWrapper) close() { // NewAddress is called by the resolver implemenetion to send addresses to gRPC. func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { select { - case <-ccr.addrCh: + case <-ccr.done: + return default: } - ccr.addrCh <- addrs + grpclog.Infof("ccResolverWrapper: sending new addresses to cc: %v", addrs) + if channelz.IsOn() { + ccr.addChannelzTraceEvent(addrs) + } + ccr.cc.handleResolvedAddrs(addrs, nil) } // NewServiceConfig is called by the resolver implemenetion to send service // configs to gRPC. func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { select { - case <-ccr.scCh: + case <-ccr.done: + return default: } - ccr.scCh <- sc + grpclog.Infof("ccResolverWrapper: got new service config: %v", sc) + ccr.cc.handleServiceConfig(sc) } func (ccr *ccResolverWrapper) addChannelzTraceEvent(addrs []resolver.Address) { diff --git a/vendor/google.golang.org/grpc/rpc_util.go b/vendor/google.golang.org/grpc/rpc_util.go index 6849e37a5..86f00e5a2 100644 --- a/vendor/google.golang.org/grpc/rpc_util.go +++ b/vendor/google.golang.org/grpc/rpc_util.go @@ -21,6 +21,7 @@ package grpc import ( "bytes" "compress/gzip" + "context" "encoding/binary" "fmt" "io" @@ -31,7 +32,6 @@ import ( "sync" "time" - "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/encoding" @@ -155,7 +155,7 @@ func (d *gzipDecompressor) Type() string { type callInfo struct { compressorType string failFast bool - stream *clientStream + stream ClientStream maxReceiveMessageSize *int maxSendMessageSize *int creds credentials.PerRPCCredentials @@ -598,13 +598,18 @@ func checkRecvPayload(pf payloadFormat, recvCompress string, haveCompressor bool return nil } -func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxReceiveMessageSize int, inPayload *stats.InPayload, compressor encoding.Compressor) ([]byte, error) { +type payloadInfo struct { + wireLength int // The compressed length got from wire. + uncompressedBytes []byte +} + +func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) ([]byte, error) { pf, d, err := p.recvMsg(maxReceiveMessageSize) if err != nil { return nil, err } - if inPayload != nil { - inPayload.WireLength = len(d) + if payInfo != nil { + payInfo.wireLength = len(d) } if st := checkRecvPayload(pf, s.RecvCompress(), compressor != nil || dc != nil); st != nil { @@ -641,20 +646,16 @@ func recvAndDecompress(p *parser, s *transport.Stream, dc Decompressor, maxRecei // For the two compressor parameters, both should not be set, but if they are, // dc takes precedence over compressor. // TODO(dfawley): wrap the old compressor/decompressor using the new API? -func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, inPayload *stats.InPayload, compressor encoding.Compressor) error { - d, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, inPayload, compressor) +func recv(p *parser, c baseCodec, s *transport.Stream, dc Decompressor, m interface{}, maxReceiveMessageSize int, payInfo *payloadInfo, compressor encoding.Compressor) error { + d, err := recvAndDecompress(p, s, dc, maxReceiveMessageSize, payInfo, compressor) if err != nil { return err } if err := c.Unmarshal(d, m); err != nil { return status.Errorf(codes.Internal, "grpc: failed to unmarshal the received message %v", err) } - if inPayload != nil { - inPayload.RecvTime = time.Now() - inPayload.Payload = m - // TODO truncate large payload. - inPayload.Data = d - inPayload.Length = len(d) + if payInfo != nil { + payInfo.uncompressedBytes = d } return nil } @@ -704,6 +705,31 @@ func Errorf(c codes.Code, format string, a ...interface{}) error { return status.Errorf(c, format, a...) } +// toRPCErr converts an error into an error from the status package. +func toRPCErr(err error) error { + if err == nil || err == io.EOF { + return err + } + if err == io.ErrUnexpectedEOF { + return status.Error(codes.Internal, err.Error()) + } + if _, ok := status.FromError(err); ok { + return err + } + switch e := err.(type) { + case transport.ConnectionError: + return status.Error(codes.Unavailable, e.Desc) + default: + switch err { + case context.DeadlineExceeded: + return status.Error(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return status.Error(codes.Canceled, err.Error()) + } + } + return status.Error(codes.Unknown, err.Error()) +} + // setCallInfoCodec should only be called after CallOptions have been applied. func setCallInfoCodec(c *callInfo) error { if c.codec != nil { diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index 920da5e01..d705d7a80 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -19,6 +19,7 @@ package grpc import ( + "context" "errors" "fmt" "io" @@ -32,7 +33,6 @@ import ( "sync/atomic" "time" - "golang.org/x/net/context" "golang.org/x/net/trace" "google.golang.org/grpc/codes" @@ -40,10 +40,12 @@ import ( "google.golang.org/grpc/encoding" "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" "google.golang.org/grpc/tap" @@ -535,7 +537,7 @@ func (s *Server) Serve(lis net.Listener) error { s.lis[ls] = true if channelz.IsOn() { - ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, "") + ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) } s.mu.Unlock() @@ -862,6 +864,30 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. }() } + binlog := binarylog.GetMethodLogger(stream.Method()) + if binlog != nil { + ctx := stream.Context() + md, _ := metadata.FromIncomingContext(ctx) + logEntry := &binarylog.ClientHeader{ + Header: md, + MethodName: stream.Method(), + PeerAddr: nil, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = deadline.Sub(time.Now()) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + if a := md[":authority"]; len(a) > 0 { + logEntry.Authority = a[0] + } + if peer, ok := peer.FromContext(ctx); ok { + logEntry.PeerAddr = peer.Addr + } + binlog.Log(logEntry) + } + // comp and cp are used for compression. decomp and dc are used for // decompression. If comp and decomp are both set, they are the same; // however they are kept separate to ensure that at most one of the @@ -898,13 +924,11 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. } } - var inPayload *stats.InPayload - if sh != nil { - inPayload = &stats.InPayload{ - RecvTime: time.Now(), - } + var payInfo *payloadInfo + if sh != nil || binlog != nil { + payInfo = &payloadInfo{} } - d, err := recvAndDecompress(&parser{r: stream}, stream, dc, s.opts.maxReceiveMessageSize, inPayload, decomp) + d, err := recvAndDecompress(&parser{r: stream}, stream, dc, s.opts.maxReceiveMessageSize, payInfo, decomp) if err != nil { if st, ok := status.FromError(err); ok { if e := t.WriteStatus(stream, st); e != nil { @@ -920,11 +944,18 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if err := s.getCodec(stream.ContentSubtype()).Unmarshal(d, v); err != nil { return status.Errorf(codes.Internal, "grpc: error unmarshalling request: %v", err) } - if inPayload != nil { - inPayload.Payload = v - inPayload.Data = d - inPayload.Length = len(d) - sh.HandleRPC(stream.Context(), inPayload) + if sh != nil { + sh.HandleRPC(stream.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: v, + Data: d, + Length: len(d), + }) + } + if binlog != nil { + binlog.Log(&binarylog.ClientMessage{ + Message: d, + }) } if trInfo != nil { trInfo.tr.LazyLog(&payload{sent: false, msg: v}, true) @@ -947,6 +978,19 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. if e := t.WriteStatus(stream, appStatus); e != nil { grpclog.Warningf("grpc: Server.processUnaryRPC failed to write status: %v", e) } + if binlog != nil { + if h, _ := stream.Header(); h.Len() > 0 { + // Only log serverHeader if there was header. Otherwise it can + // be trailer only. + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + } + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } return appErr } if trInfo != nil { @@ -971,8 +1015,27 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. panic(fmt.Sprintf("grpc: Unexpected error (%T) from sendResponse: %v", st, st)) } } + if binlog != nil { + h, _ := stream.Header() + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } return err } + if binlog != nil { + h, _ := stream.Header() + binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + binlog.Log(&binarylog.ServerMessage{ + Message: reply, + }) + } if channelz.IsOn() { t.IncrMsgSent() } @@ -982,7 +1045,14 @@ func (s *Server) processUnaryRPC(t transport.ServerTransport, stream *transport. // TODO: Should we be logging if writing status failed here, like above? // Should the logging be in WriteStatus? Should we ignore the WriteStatus // error or allow the stats handler to see it? - return t.WriteStatus(stream, status.New(codes.OK, "")) + err = t.WriteStatus(stream, status.New(codes.OK, "")) + if binlog != nil { + binlog.Log(&binarylog.ServerTrailer{ + Trailer: stream.Trailer(), + Err: appErr, + }) + } + return err } func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transport.Stream, srv *service, sd *StreamDesc, trInfo *traceInfo) (err error) { @@ -1027,6 +1097,29 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp statsHandler: sh, } + ss.binlog = binarylog.GetMethodLogger(stream.Method()) + if ss.binlog != nil { + md, _ := metadata.FromIncomingContext(ctx) + logEntry := &binarylog.ClientHeader{ + Header: md, + MethodName: stream.Method(), + PeerAddr: nil, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = deadline.Sub(time.Now()) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + if a := md[":authority"]; len(a) > 0 { + logEntry.Authority = a[0] + } + if peer, ok := peer.FromContext(ss.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + ss.binlog.Log(logEntry) + } + // If dc is set and matches the stream's compression, use it. Otherwise, try // to find a matching registered compressor for decomp. if rc := stream.RecvCompress(); s.opts.dc != nil && s.opts.dc.Type() == rc { @@ -1096,6 +1189,12 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.mu.Unlock() } t.WriteStatus(ss.s, appStatus) + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + }) + } // TODO: Should we log an error from WriteStatus here and below? return appErr } @@ -1104,7 +1203,14 @@ func (s *Server) processStreamingRPC(t transport.ServerTransport, stream *transp ss.trInfo.tr.LazyLog(stringer("OK"), false) ss.mu.Unlock() } - return t.WriteStatus(ss.s, status.New(codes.OK, "")) + err = t.WriteStatus(ss.s, status.New(codes.OK, "")) + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ServerTrailer{ + Trailer: ss.s.Trailer(), + Err: appErr, + }) + } + return err } func (s *Server) handleStream(t transport.ServerTransport, stream *transport.Stream, trInfo *traceInfo) { diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go index a305fe0a4..162857e20 100644 --- a/vendor/google.golang.org/grpc/service_config.go +++ b/vendor/google.golang.org/grpc/service_config.go @@ -96,6 +96,15 @@ type ServiceConfig struct { // If token_count is less than or equal to maxTokens / 2, then RPCs will not // be retried and hedged RPCs will not be sent. retryThrottling *retryThrottlingPolicy + // healthCheckConfig must be set as one of the requirement to enable LB channel + // health check. + healthCheckConfig *healthCheckConfig +} + +// healthCheckConfig defines the go-native version of the LB channel health check config. +type healthCheckConfig struct { + // serviceName is the service name to use in the health-checking request. + ServiceName string } // retryPolicy defines the go-native version of the retry policy defined by the @@ -226,6 +235,7 @@ type jsonSC struct { LoadBalancingPolicy *string MethodConfig *[]jsonMC RetryThrottling *retryThrottlingPolicy + HealthCheckConfig *healthCheckConfig } func parseServiceConfig(js string) (ServiceConfig, error) { @@ -239,9 +249,10 @@ func parseServiceConfig(js string) (ServiceConfig, error) { return ServiceConfig{}, err } sc := ServiceConfig{ - LB: rsc.LoadBalancingPolicy, - Methods: make(map[string]MethodConfig), - retryThrottling: rsc.RetryThrottling, + LB: rsc.LoadBalancingPolicy, + Methods: make(map[string]MethodConfig), + retryThrottling: rsc.RetryThrottling, + healthCheckConfig: rsc.HealthCheckConfig, } if rsc.MethodConfig == nil { return sc, nil diff --git a/vendor/google.golang.org/grpc/stats/handlers.go b/vendor/google.golang.org/grpc/stats/handlers.go index 05b384c69..dc03731e4 100644 --- a/vendor/google.golang.org/grpc/stats/handlers.go +++ b/vendor/google.golang.org/grpc/stats/handlers.go @@ -19,9 +19,8 @@ package stats import ( + "context" "net" - - "golang.org/x/net/context" ) // ConnTagInfo defines the relevant information needed by connection context tagger. diff --git a/vendor/google.golang.org/grpc/stats/stats.go b/vendor/google.golang.org/grpc/stats/stats.go index 3f13190a0..84f77dafa 100644 --- a/vendor/google.golang.org/grpc/stats/stats.go +++ b/vendor/google.golang.org/grpc/stats/stats.go @@ -24,10 +24,9 @@ package stats // import "google.golang.org/grpc/stats" import ( + "context" "net" "time" - - "golang.org/x/net/context" ) // RPCStats contains stats information about RPCs. diff --git a/vendor/google.golang.org/grpc/status/go16.go b/vendor/google.golang.org/grpc/status/go16.go deleted file mode 100644 index e59b53e82..000000000 --- a/vendor/google.golang.org/grpc/status/go16.go +++ /dev/null @@ -1,42 +0,0 @@ -// +build go1.6,!go1.7 - -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package status - -import ( - "golang.org/x/net/context" - "google.golang.org/grpc/codes" -) - -// FromContextError converts a context error into a Status. It returns a -// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is -// non-nil and not a context error. -func FromContextError(err error) *Status { - switch err { - case nil: - return New(codes.OK, "") - case context.DeadlineExceeded: - return New(codes.DeadlineExceeded, err.Error()) - case context.Canceled: - return New(codes.Canceled, err.Error()) - default: - return New(codes.Unknown, err.Error()) - } -} diff --git a/vendor/google.golang.org/grpc/status/go17.go b/vendor/google.golang.org/grpc/status/go17.go deleted file mode 100644 index 090215149..000000000 --- a/vendor/google.golang.org/grpc/status/go17.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build go1.7 - -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package status - -import ( - "context" - - netctx "golang.org/x/net/context" - "google.golang.org/grpc/codes" -) - -// FromContextError converts a context error into a Status. It returns a -// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is -// non-nil and not a context error. -func FromContextError(err error) *Status { - switch err { - case nil: - return New(codes.OK, "") - case context.DeadlineExceeded, netctx.DeadlineExceeded: - return New(codes.DeadlineExceeded, err.Error()) - case context.Canceled, netctx.Canceled: - return New(codes.Canceled, err.Error()) - default: - return New(codes.Unknown, err.Error()) - } -} diff --git a/vendor/google.golang.org/grpc/status/status.go b/vendor/google.golang.org/grpc/status/status.go index 897321bab..ed36681bb 100644 --- a/vendor/google.golang.org/grpc/status/status.go +++ b/vendor/google.golang.org/grpc/status/status.go @@ -28,6 +28,7 @@ package status import ( + "context" "errors" "fmt" @@ -191,3 +192,19 @@ func Code(err error) codes.Code { } return codes.Unknown } + +// FromContextError converts a context error into a Status. It returns a +// Status with codes.OK if err is nil, or a Status with codes.Unknown if err is +// non-nil and not a context error. +func FromContextError(err error) *Status { + switch err { + case nil: + return New(codes.OK, "") + case context.DeadlineExceeded: + return New(codes.DeadlineExceeded, err.Error()) + case context.Canceled: + return New(codes.Canceled, err.Error()) + default: + return New(codes.Unknown, err.Error()) + } +} diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index b71eb3112..0c266d6f9 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -19,6 +19,7 @@ package grpc import ( + "context" "errors" "io" "math" @@ -26,16 +27,18 @@ import ( "sync" "time" - "golang.org/x/net/context" "golang.org/x/net/trace" "google.golang.org/grpc/balancer" "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/encoding" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/binarylog" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/transport" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) @@ -82,7 +85,8 @@ type ClientStream interface { // stream.Recv has returned a non-nil error (including io.EOF). Trailer() metadata.MD // CloseSend closes the send direction of the stream. It closes the stream - // when non-nil error is met. + // when non-nil error is met. It is also not safe to call CloseSend + // concurrently with SendMsg. CloseSend() error // Context returns the context for this stream. // @@ -105,7 +109,8 @@ type ClientStream interface { // // It is safe to have a goroutine calling SendMsg and another goroutine // calling RecvMsg on the same stream at the same time, but it is not safe - // to call SendMsg on the same stream in different goroutines. + // to call SendMsg on the same stream in different goroutines. It is also + // not safe to call CloseSend concurrently with SendMsg. SendMsg(m interface{}) error // RecvMsg blocks until it receives a message into m or the stream is // done. It returns io.EOF when the stream completes successfully. On @@ -160,6 +165,11 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth }() } c := defaultCallInfo() + // Provide an opportunity for the first RPC to see the first service config + // provided by the resolver. + if err := cc.waitForResolvedAddrs(ctx); err != nil { + return nil, err + } mc := cc.GetMethodConfig(method) if mc.WaitForReady != nil { c.failFast = !*mc.WaitForReady @@ -262,6 +272,7 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth if !cc.dopts.disableRetry { cs.retryThrottler = cc.retryThrottler.Load().(*retryThrottler) } + cs.binlog = binarylog.GetMethodLogger(method) cs.callInfo.stream = cs // Only this initial attempt has stats/tracing. @@ -277,6 +288,23 @@ func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth return nil, err } + if cs.binlog != nil { + md, _ := metadata.FromOutgoingContext(ctx) + logEntry := &binarylog.ClientHeader{ + OnClientSide: true, + Header: md, + MethodName: method, + Authority: cs.cc.authority, + } + if deadline, ok := ctx.Deadline(); ok { + logEntry.Timeout = deadline.Sub(time.Now()) + if logEntry.Timeout < 0 { + logEntry.Timeout = 0 + } + } + cs.binlog.Log(logEntry) + } + if desc != unaryStreamDesc { // Listen on cc and stream contexts to cleanup when the user closes the // ClientConn or cancels the stream context. In all other cases, an error @@ -350,6 +378,15 @@ type clientStream struct { retryThrottler *retryThrottler // The throttler active when the RPC began. + binlog *binarylog.MethodLogger // Binary logger, can be nil. + // serverHeaderBinlogged is a boolean for whether server header has been + // logged. Server header will be logged when the first time one of those + // happens: stream.Header(), stream.Recv(). + // + // It's only read and used by Recv() and Header(), so it doesn't need to be + // synchronized. + serverHeaderBinlogged bool + mu sync.Mutex firstAttempt bool // if true, transparent retry is valid numRetries int // exclusive of transparent retry attempt(s) @@ -561,6 +598,20 @@ func (cs *clientStream) Header() (metadata.MD, error) { }, cs.commitAttemptLocked) if err != nil { cs.finish(err) + return nil, err + } + if cs.binlog != nil && !cs.serverHeaderBinlogged { + // Only log if binary log is on and header has not been logged. + logEntry := &binarylog.ServerHeader{ + OnClientSide: true, + Header: m, + PeerAddr: nil, + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + cs.binlog.Log(logEntry) + cs.serverHeaderBinlogged = true } return m, err } @@ -633,6 +684,7 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { if len(payload) > *cs.callInfo.maxSendMessageSize { return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payload), *cs.callInfo.maxSendMessageSize) } + msgBytes := data // Store the pointer before setting to nil. For binary logging. op := func(a *csAttempt) error { err := a.sendMsg(m, hdr, payload, data) // nil out the message and uncomp when replaying; they are only needed for @@ -640,16 +692,53 @@ func (cs *clientStream) SendMsg(m interface{}) (err error) { m, data = nil, nil return err } - return cs.withRetry(op, func() { cs.bufferForRetryLocked(len(hdr)+len(payload), op) }) + err = cs.withRetry(op, func() { cs.bufferForRetryLocked(len(hdr)+len(payload), op) }) + if cs.binlog != nil && err == nil { + cs.binlog.Log(&binarylog.ClientMessage{ + OnClientSide: true, + Message: msgBytes, + }) + } + return } func (cs *clientStream) RecvMsg(m interface{}) error { + if cs.binlog != nil && !cs.serverHeaderBinlogged { + // Call Header() to binary log header if it's not already logged. + cs.Header() + } + var recvInfo *payloadInfo + if cs.binlog != nil { + recvInfo = &payloadInfo{} + } err := cs.withRetry(func(a *csAttempt) error { - return a.recvMsg(m) + return a.recvMsg(m, recvInfo) }, cs.commitAttemptLocked) + if cs.binlog != nil && err == nil { + cs.binlog.Log(&binarylog.ServerMessage{ + OnClientSide: true, + Message: recvInfo.uncompressedBytes, + }) + } if err != nil || !cs.desc.ServerStreams { // err != nil or non-server-streaming indicates end of stream. cs.finish(err) + + if cs.binlog != nil { + // finish will not log Trailer. Log Trailer here. + logEntry := &binarylog.ServerTrailer{ + OnClientSide: true, + Trailer: cs.Trailer(), + Err: err, + } + if logEntry.Err == io.EOF { + logEntry.Err = nil + } + if peer, ok := peer.FromContext(cs.Context()); ok { + logEntry.PeerAddr = peer.Addr + } + cs.binlog.Log(logEntry) + } } return err } @@ -669,6 +758,11 @@ func (cs *clientStream) CloseSend() error { return nil } cs.withRetry(op, func() { cs.bufferForRetryLocked(0, op) }) + if cs.binlog != nil { + cs.binlog.Log(&binarylog.ClientHalfClose{ + OnClientSide: true, + }) + } // We never returned an error here for reasons. return nil } @@ -686,6 +780,16 @@ func (cs *clientStream) finish(err error) { cs.finished = true cs.commitAttemptLocked() cs.mu.Unlock() + // For binary logging. only log cancel in finish (could be caused by RPC ctx + // canceled or ClientConn closed). Trailer will be logged in RecvMsg. + // + // Only one of cancel or trailer needs to be logged. In the cases where + // users don't call RecvMsg, users must have already canceled the RPC. + if cs.binlog != nil && status.Code(err) == codes.Canceled { + cs.binlog.Log(&binarylog.Cancel{ + OnClientSide: true, + }) + } if err == nil { cs.retryThrottler.successfulRPC() } @@ -735,14 +839,12 @@ func (a *csAttempt) sendMsg(m interface{}, hdr, payld, data []byte) error { return nil } -func (a *csAttempt) recvMsg(m interface{}) (err error) { +func (a *csAttempt) recvMsg(m interface{}, payInfo *payloadInfo) (err error) { cs := a.cs - var inPayload *stats.InPayload - if a.statsHandler != nil { - inPayload = &stats.InPayload{ - Client: true, - } + if a.statsHandler != nil && payInfo == nil { + payInfo = &payloadInfo{} } + if !a.decompSet { // Block until we receive headers containing received message encoding. if ct := a.s.RecvCompress(); ct != "" && ct != encoding.Identity { @@ -759,7 +861,7 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { // Only initialize this state once per stream. a.decompSet = true } - err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, inPayload, a.decomp) + err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, payInfo, a.decomp) if err != nil { if err == io.EOF { if statusErr := a.s.Status().Err(); statusErr != nil { @@ -776,8 +878,15 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { } a.mu.Unlock() } - if inPayload != nil { - a.statsHandler.HandleRPC(cs.ctx, inPayload) + if a.statsHandler != nil { + a.statsHandler.HandleRPC(cs.ctx, &stats.InPayload{ + Client: true, + RecvTime: time.Now(), + Payload: m, + // TODO truncate large payload. + Data: payInfo.uncompressedBytes, + Length: len(payInfo.uncompressedBytes), + }) } if channelz.IsOn() { a.t.IncrMsgRecv() @@ -786,7 +895,6 @@ func (a *csAttempt) recvMsg(m interface{}) (err error) { // Subsequent messages should be received by subsequent RecvMsg calls. return nil } - // Special handling for non-server-stream rpcs. // This recv expects EOF or errors, so we don't collect inPayload. err = recv(a.p, cs.codec, a.s, a.dc, m, *cs.callInfo.maxReceiveMessageSize, nil, a.decomp) @@ -850,6 +958,299 @@ func (a *csAttempt) finish(err error) { a.mu.Unlock() } +func (ac *addrConn) newClientStream(ctx context.Context, desc *StreamDesc, method string, t transport.ClientTransport, opts ...CallOption) (_ ClientStream, err error) { + ac.mu.Lock() + if ac.transport != t { + ac.mu.Unlock() + return nil, status.Error(codes.Canceled, "the provided transport is no longer valid to use") + } + // transition to CONNECTING state when an attempt starts + if ac.state != connectivity.Connecting { + ac.updateConnectivityState(connectivity.Connecting) + ac.cc.handleSubConnStateChange(ac.acbw, ac.state) + } + ac.mu.Unlock() + + if t == nil { + // TODO: return RPC error here? + return nil, errors.New("transport provided is nil") + } + // defaultCallInfo contains unnecessary info(i.e. failfast, maxRetryRPCBufferSize), so we just initialize an empty struct. + c := &callInfo{} + + for _, o := range opts { + if err := o.before(c); err != nil { + return nil, toRPCErr(err) + } + } + c.maxReceiveMessageSize = getMaxSize(nil, c.maxReceiveMessageSize, defaultClientMaxReceiveMessageSize) + c.maxSendMessageSize = getMaxSize(nil, c.maxSendMessageSize, defaultServerMaxSendMessageSize) + + // Possible context leak: + // The cancel function for the child context we create will only be called + // when RecvMsg returns a non-nil error, if the ClientConn is closed, or if + // an error is generated by SendMsg. + // https://github.com/grpc/grpc-go/issues/1818. + ctx, cancel := context.WithCancel(ctx) + defer func() { + if err != nil { + cancel() + } + }() + + if err := setCallInfoCodec(c); err != nil { + return nil, err + } + + callHdr := &transport.CallHdr{ + Host: ac.cc.authority, + Method: method, + ContentSubtype: c.contentSubtype, + } + + // Set our outgoing compression according to the UseCompressor CallOption, if + // set. In that case, also find the compressor from the encoding package. + // Otherwise, use the compressor configured by the WithCompressor DialOption, + // if set. + var cp Compressor + var comp encoding.Compressor + if ct := c.compressorType; ct != "" { + callHdr.SendCompress = ct + if ct != encoding.Identity { + comp = encoding.GetCompressor(ct) + if comp == nil { + return nil, status.Errorf(codes.Internal, "grpc: Compressor is not installed for requested grpc-encoding %q", ct) + } + } + } else if ac.cc.dopts.cp != nil { + callHdr.SendCompress = ac.cc.dopts.cp.Type() + cp = ac.cc.dopts.cp + } + if c.creds != nil { + callHdr.Creds = c.creds + } + + as := &addrConnStream{ + callHdr: callHdr, + ac: ac, + ctx: ctx, + cancel: cancel, + opts: opts, + callInfo: c, + desc: desc, + codec: c.codec, + cp: cp, + comp: comp, + t: t, + } + + as.callInfo.stream = as + s, err := as.t.NewStream(as.ctx, as.callHdr) + if err != nil { + err = toRPCErr(err) + return nil, err + } + as.s = s + as.p = &parser{r: s} + ac.incrCallsStarted() + if desc != unaryStreamDesc { + // Listen on cc and stream contexts to cleanup when the user closes the + // ClientConn or cancels the stream context. In all other cases, an error + // should already be injected into the recv buffer by the transport, which + // the client will eventually receive, and then we will cancel the stream's + // context in clientStream.finish. + go func() { + select { + case <-ac.ctx.Done(): + as.finish(status.Error(codes.Canceled, "grpc: the SubConn is closing")) + case <-ctx.Done(): + as.finish(toRPCErr(ctx.Err())) + } + }() + } + return as, nil +} + +type addrConnStream struct { + s *transport.Stream + ac *addrConn + callHdr *transport.CallHdr + cancel context.CancelFunc + opts []CallOption + callInfo *callInfo + t transport.ClientTransport + ctx context.Context + sentLast bool + desc *StreamDesc + codec baseCodec + cp Compressor + comp encoding.Compressor + decompSet bool + dc Decompressor + decomp encoding.Compressor + p *parser + done func(balancer.DoneInfo) + mu sync.Mutex + finished bool +} + +func (as *addrConnStream) Header() (metadata.MD, error) { + m, err := as.s.Header() + if err != nil { + as.finish(toRPCErr(err)) + } + return m, err +} + +func (as *addrConnStream) Trailer() metadata.MD { + return as.s.Trailer() +} + +func (as *addrConnStream) CloseSend() error { + if as.sentLast { + // TODO: return an error and finish the stream instead, due to API misuse? + return nil + } + as.sentLast = true + + as.t.Write(as.s, nil, nil, &transport.Options{Last: true}) + // Always return nil; io.EOF is the only error that might make sense + // instead, but there is no need to signal the client to call RecvMsg + // as the only use left for the stream after CloseSend is to call + // RecvMsg. This also matches historical behavior. + return nil +} + +func (as *addrConnStream) Context() context.Context { + return as.s.Context() +} + +func (as *addrConnStream) SendMsg(m interface{}) (err error) { + defer func() { + if err != nil && err != io.EOF { + // Call finish on the client stream for errors generated by this SendMsg + // call, as these indicate problems created by this client. (Transport + // errors are converted to an io.EOF error in csAttempt.sendMsg; the real + // error will be returned from RecvMsg eventually in that case, or be + // retried.) + as.finish(err) + } + }() + if as.sentLast { + return status.Errorf(codes.Internal, "SendMsg called after CloseSend") + } + if !as.desc.ClientStreams { + as.sentLast = true + } + data, err := encode(as.codec, m) + if err != nil { + return err + } + compData, err := compress(data, as.cp, as.comp) + if err != nil { + return err + } + hdr, payld := msgHeader(data, compData) + // TODO(dfawley): should we be checking len(data) instead? + if len(payld) > *as.callInfo.maxSendMessageSize { + return status.Errorf(codes.ResourceExhausted, "trying to send message larger than max (%d vs. %d)", len(payld), *as.callInfo.maxSendMessageSize) + } + + if err := as.t.Write(as.s, hdr, payld, &transport.Options{Last: !as.desc.ClientStreams}); err != nil { + if !as.desc.ClientStreams { + // For non-client-streaming RPCs, we return nil instead of EOF on error + // because the generated code requires it. finish is not called; RecvMsg() + // will call it with the stream's status independently. + return nil + } + return io.EOF + } + + if channelz.IsOn() { + as.t.IncrMsgSent() + } + return nil +} + +func (as *addrConnStream) RecvMsg(m interface{}) (err error) { + defer func() { + if err != nil || !as.desc.ServerStreams { + // err != nil or non-server-streaming indicates end of stream. + as.finish(err) + } + }() + + if !as.decompSet { + // Block until we receive headers containing received message encoding. + if ct := as.s.RecvCompress(); ct != "" && ct != encoding.Identity { + if as.dc == nil || as.dc.Type() != ct { + // No configured decompressor, or it does not match the incoming + // message encoding; attempt to find a registered compressor that does. + as.dc = nil + as.decomp = encoding.GetCompressor(ct) + } + } else { + // No compression is used; disable our decompressor. + as.dc = nil + } + // Only initialize this state once per stream. + as.decompSet = true + } + err = recv(as.p, as.codec, as.s, as.dc, m, *as.callInfo.maxReceiveMessageSize, nil, as.decomp) + if err != nil { + if err == io.EOF { + if statusErr := as.s.Status().Err(); statusErr != nil { + return statusErr + } + return io.EOF // indicates successful end of stream. + } + return toRPCErr(err) + } + + if channelz.IsOn() { + as.t.IncrMsgRecv() + } + if as.desc.ServerStreams { + // Subsequent messages should be received by subsequent RecvMsg calls. + return nil + } + + // Special handling for non-server-stream rpcs. + // This recv expects EOF or errors, so we don't collect inPayload. + err = recv(as.p, as.codec, as.s, as.dc, m, *as.callInfo.maxReceiveMessageSize, nil, as.decomp) + if err == nil { + return toRPCErr(errors.New("grpc: client streaming protocol violation: get , want ")) + } + if err == io.EOF { + return as.s.Status().Err() // non-server streaming Recv returns nil on success + } + return toRPCErr(err) +} + +func (as *addrConnStream) finish(err error) { + as.mu.Lock() + if as.finished { + as.mu.Unlock() + return + } + as.finished = true + if err == io.EOF { + // Ending a stream with EOF indicates a success. + err = nil + } + if as.s != nil { + as.t.CloseStream(as.s, err) + } + + if err != nil { + as.ac.incrCallsFailed() + } else { + as.ac.incrCallsSucceeded() + } + as.cancel() + as.mu.Unlock() +} + // ServerStream defines the server-side behavior of a streaming RPC. // // All errors returned from ServerStream methods are compatible with the @@ -916,6 +1317,15 @@ type serverStream struct { statsHandler stats.Handler + binlog *binarylog.MethodLogger + // serverHeaderBinlogged indicates whether server header has been logged. It + // will happen when one of the following two happens: stream.SendHeader(), + // stream.Send(). + // + // It's only checked in send and sendHeader, doesn't need to be + // synchronized. + serverHeaderBinlogged bool + mu sync.Mutex // protects trInfo.tr after the service handler runs. } @@ -931,7 +1341,15 @@ func (ss *serverStream) SetHeader(md metadata.MD) error { } func (ss *serverStream) SendHeader(md metadata.MD) error { - return ss.t.WriteHeader(ss.s, md) + err := ss.t.WriteHeader(ss.s, md) + if ss.binlog != nil && !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + ss.binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + ss.serverHeaderBinlogged = true + } + return err } func (ss *serverStream) SetTrailer(md metadata.MD) { @@ -958,6 +1376,12 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err != nil && err != io.EOF { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. } if channelz.IsOn() && err == nil { ss.t.IncrMsgSent() @@ -979,6 +1403,18 @@ func (ss *serverStream) SendMsg(m interface{}) (err error) { if err := ss.t.Write(ss.s, hdr, payload, &transport.Options{Last: false}); err != nil { return toRPCErr(err) } + if ss.binlog != nil { + if !ss.serverHeaderBinlogged { + h, _ := ss.s.Header() + ss.binlog.Log(&binarylog.ServerHeader{ + Header: h, + }) + ss.serverHeaderBinlogged = true + } + ss.binlog.Log(&binarylog.ServerMessage{ + Message: data, + }) + } if ss.statsHandler != nil { ss.statsHandler.HandleRPC(ss.s.Context(), outPayload(false, m, data, payload, time.Now())) } @@ -1002,17 +1438,26 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { if err != nil && err != io.EOF { st, _ := status.FromError(toRPCErr(err)) ss.t.WriteStatus(ss.s, st) + // Non-user specified status was sent out. This should be an error + // case (as a server side Cancel maybe). + // + // This is not handled specifically now. User will return a final + // status from the service handler, we will log that error instead. + // This behavior is similar to an interceptor. } if channelz.IsOn() && err == nil { ss.t.IncrMsgRecv() } }() - var inPayload *stats.InPayload - if ss.statsHandler != nil { - inPayload = &stats.InPayload{} + var payInfo *payloadInfo + if ss.statsHandler != nil || ss.binlog != nil { + payInfo = &payloadInfo{} } - if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, inPayload, ss.decomp); err != nil { + if err := recv(ss.p, ss.codec, ss.s, ss.dc, m, ss.maxReceiveMessageSize, payInfo, ss.decomp); err != nil { if err == io.EOF { + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ClientHalfClose{}) + } return err } if err == io.ErrUnexpectedEOF { @@ -1020,8 +1465,19 @@ func (ss *serverStream) RecvMsg(m interface{}) (err error) { } return toRPCErr(err) } - if inPayload != nil { - ss.statsHandler.HandleRPC(ss.s.Context(), inPayload) + if ss.statsHandler != nil { + ss.statsHandler.HandleRPC(ss.s.Context(), &stats.InPayload{ + RecvTime: time.Now(), + Payload: m, + // TODO truncate large payload. + Data: payInfo.uncompressedBytes, + Length: len(payInfo.uncompressedBytes), + }) + } + if ss.binlog != nil { + ss.binlog.Log(&binarylog.ClientMessage{ + Message: payInfo.uncompressedBytes, + }) } return nil } diff --git a/vendor/google.golang.org/grpc/tap/tap.go b/vendor/google.golang.org/grpc/tap/tap.go index 22b8fb50d..584360f68 100644 --- a/vendor/google.golang.org/grpc/tap/tap.go +++ b/vendor/google.golang.org/grpc/tap/tap.go @@ -21,7 +21,7 @@ package tap import ( - "golang.org/x/net/context" + "context" ) // Info defines the relevant information needed by the handles. diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index d8e0287ef..260f27c88 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.16.0" +const Version = "1.17.0"