From 6600ab49747986eca7154dc6ac5ee7378e2210e6 Mon Sep 17 00:00:00 2001 From: Tobias Schottdorf Date: Tue, 16 Jul 2019 11:15:55 +0200 Subject: [PATCH] raft: internally support joint consensus This commit introduces machinery to safely apply joint consensus configuration changes to Raft. The main contribution is the new package, `confchange`, which offers the primitives `Simple`, `EnterJoint`, and `LeaveJoint`. The first two take a list of configuration changes. `Simple` only declares success if these configuration changes (applied atomically) change the set of voters by at most one (i.e. it's fine to add or remove any number of learners, but change only one voter). `EnterJoint` makes the configuration joint and then applies the changes to it, in preparation of the caller returning later and transitioning out of the joint config into the final desired configuration via `LeaveJoint()`. This commit streamlines the conversion between voters and learners, which is now generally allowed whenever the above conditions are upheld (i.e. it's not possible to demote a voter and add a new voter in the context of a Simple configuration change, but it is possible via EnterJoint). Previously, we had the artificial restriction that a voter could not be demoted to a learner, but had to be removed first. Even though demoting a learner is generally less useful than promoting a learner (the latter is used to catch up future voters), demotions could see use in improved handling of temporary node unavailability, where it is desired to remove voting power from a down node, but to preserve its data should it return. An additional change that was made in this commit is to prevent the use of empty commit quorums, which was previously possible but for no good reason; this: Closes #10884. The work left to do in a future PR is to actually expose joint configurations to the applications using Raft. This will entail mostly API design and the addition of suitable testing, which to be carried out ergonomically is likely to motivate a larger refactor. Touches #7625. --- go.mod | 6 +- go.sum | 14 +- raft/confchange/confchange.go | 411 ++++++++++++++++++ raft/confchange/datadriven_test.go | 105 +++++ raft/confchange/quick_test.go | 169 +++++++ .../confchange/testdata/joint_idempotency.txt | 23 + .../testdata/joint_learners_next.txt | 24 + raft/confchange/testdata/joint_safety.txt | 81 ++++ .../testdata/simple_idempotency.txt | 69 +++ .../testdata/simple_promote_demote.txt | 60 +++ raft/confchange/testdata/simple_safety.txt | 64 +++ raft/confchange/testdata/update.txt | 23 + raft/confchange/testdata/zero.txt | 6 + raft/raft.go | 71 +-- raft/raft_test.go | 66 ++- raft/tracker/progress.go | 20 + raft/tracker/tracker.go | 103 ++--- .../cockroachdb/datadriven/datadriven.go | 57 +-- vendor/github.com/kr/pty/pty_darwin.go | 15 +- vendor/github.com/kr/pty/pty_dragonfly.go | 12 +- vendor/github.com/kr/pty/pty_freebsd.go | 19 +- vendor/github.com/kr/pty/pty_linux.go | 9 +- vendor/github.com/kr/pty/pty_openbsd.go | 33 ++ vendor/github.com/kr/pty/pty_unsupported.go | 2 +- vendor/github.com/kr/pty/types_openbsd.go | 14 + vendor/github.com/kr/pty/util.go | 47 +- .../github.com/kr/pty/ztypes_openbsd_amd64.go | 13 + vendor/github.com/kr/text/License | 19 + vendor/github.com/kr/text/doc.go | 3 + vendor/github.com/kr/text/indent.go | 74 ++++ vendor/github.com/kr/text/wrap.go | 86 ++++ vendor/github.com/motomux/pretty/License | 21 + vendor/github.com/motomux/pretty/diff.go | 273 ++++++++++++ vendor/github.com/motomux/pretty/formatter.go | 328 ++++++++++++++ vendor/github.com/motomux/pretty/pretty.go | 108 +++++ vendor/github.com/motomux/pretty/zero.go | 41 ++ 36 files changed, 2296 insertions(+), 193 deletions(-) create mode 100644 raft/confchange/confchange.go create mode 100644 raft/confchange/datadriven_test.go create mode 100644 raft/confchange/quick_test.go create mode 100644 raft/confchange/testdata/joint_idempotency.txt create mode 100644 raft/confchange/testdata/joint_learners_next.txt create mode 100644 raft/confchange/testdata/joint_safety.txt create mode 100644 raft/confchange/testdata/simple_idempotency.txt create mode 100644 raft/confchange/testdata/simple_promote_demote.txt create mode 100644 raft/confchange/testdata/simple_safety.txt create mode 100644 raft/confchange/testdata/update.txt create mode 100644 raft/confchange/testdata/zero.txt create mode 100644 vendor/github.com/kr/pty/pty_openbsd.go create mode 100644 vendor/github.com/kr/pty/types_openbsd.go create mode 100644 vendor/github.com/kr/pty/ztypes_openbsd_amd64.go create mode 100644 vendor/github.com/kr/text/License create mode 100644 vendor/github.com/kr/text/doc.go create mode 100644 vendor/github.com/kr/text/indent.go create mode 100644 vendor/github.com/kr/text/wrap.go create mode 100644 vendor/github.com/motomux/pretty/License create mode 100644 vendor/github.com/motomux/pretty/diff.go create mode 100644 vendor/github.com/motomux/pretty/formatter.go create mode 100644 vendor/github.com/motomux/pretty/pretty.go create mode 100644 vendor/github.com/motomux/pretty/zero.go diff --git a/go.mod b/go.mod index 31eabdb8a4a4..001a92335282 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.etcd.io/etcd require ( github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/bgentry/speakeasy v0.1.0 - github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238 + github.com/cockroachdb/datadriven v0.0.0-20190711114415-5fd2960cd810 github.com/coreos/go-semver v0.2.0 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf @@ -24,13 +24,15 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jonboulle/clockwork v0.1.0 github.com/json-iterator/go v1.1.5 - github.com/kr/pty v1.0.0 + github.com/kr/pretty v0.1.0 // indirect + github.com/kr/pty v1.1.1 github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect github.com/mattn/go-runewidth v0.0.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 + github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 github.com/onsi/gomega v1.4.2 // indirect github.com/pkg/errors v0.8.0 // indirect diff --git a/go.sum b/go.sum index 213796aa75c4..27a2afa063b7 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238 h1:uNljlOxtOHrPnRoPPx+JanqjAGZpNiqAGVBfGskd/pg= -github.com/cockroachdb/datadriven v0.0.0-20190531201743-edce55837238/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20190711114415-5fd2960cd810 h1:c3OcmacTfydv/0dp84SWDUb9vOMcf6td5TQyY9+opgM= +github.com/cockroachdb/datadriven v0.0.0-20190711114415-5fd2960cd810/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= @@ -48,8 +48,12 @@ github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/kr/pty v1.0.0 h1:jR04h3bskdxb8xt+5B6MoxPwDhMCe0oEgxug4Ca1YSA= -github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= @@ -62,6 +66,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d h1:LznySqW8MqVeFh+pW6rOkFdld9QQ7jRydBKKM6jyPVI= +github.com/motomux/pretty v0.0.0-20161209205251-b2aad2c9a95d/go.mod h1:u3hJ0kqCQu/cPpsu3RbCOPZ0d7V3IjPjv1adNRleM9I= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= diff --git a/raft/confchange/confchange.go b/raft/confchange/confchange.go new file mode 100644 index 000000000000..1881811e818a --- /dev/null +++ b/raft/confchange/confchange.go @@ -0,0 +1,411 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "errors" + "fmt" + "strings" + + "go.etcd.io/etcd/raft/quorum" + pb "go.etcd.io/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/tracker" +) + +// Changer facilitates configuration changes. It exposes methods to handle +// simple and joint consensus while performing the proper validation that allows +// refusing invalid configuration changes before they affect the active +// configuration. +type Changer struct { + Tracker tracker.ProgressTracker + LastIndex uint64 +} + +// EnterJoint verifies that the outgoing (=right) majority config of the joint +// config is empty and initializes it with a copy of the incoming (=left) +// majority config. That is, it transitions from +// +// (1 2 3)&&() +// to +// (1 2 3)&&(1 2 3). +// +// The supplied ConfChanges are then applied to the incoming majority config, +// resulting in a joint configuration that in terms of the Raft thesis[1] +// (Section 4.3) corresponds to `C_{new,old}`. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) EnterJoint(ccs ...pb.ConfChange) (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("config is already joint") + return c.err(err) + } + if len(cfg.Voters[0]) == 0 { + // We allow adding nodes to an empty config for convenience (testing and + // bootstrap), but you can't enter a joint state. + err := errors.New("can't make a zero-voter config joint") + return c.err(err) + } + cfg.Voters[1] = quorum.MajorityConfig{} + for id := range cfg.Voters[0] { + cfg.Voters[1][id] = struct{}{} + } + + if err := c.apply(&cfg, prs, ccs...); err != nil { + return c.err(err) + } + + return checkAndReturn(cfg, prs) +} + +// LeaveJoint transitions out of a joint configuration. It is an error to call +// this method if the configuration is not joint, i.e. if the outgoing majority +// config Voters[1] is empty. +// +// The outgoing majority config of the joint configuration will be removed, +// that is, the incoming config is promoted as the sole decision maker. In the +// notation of the Raft thesis[1] (Section 4.3), this method transitions from +// `C_{new,old}` into `C_new`. +// +// At the same time, any staged learners (LearnersNext) the addition of which +// was held back by an overlapping voter in the former outgoing config will be +// inserted into Learners. +// +// [1]: https://github.com/ongardie/dissertation/blob/master/online-trim.pdf +func (c Changer) LeaveJoint() (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if !joint(cfg) { + err := errors.New("can't leave a non-joint config") + return c.err(err) + } + if len(cfg.Voters[1]) == 0 { + err := fmt.Errorf("configuration is not joint: %v", cfg) + return c.err(err) + } + for id := range cfg.LearnersNext { + nilAwareAdd(&cfg.Learners, id) + prs[id].IsLearner = true + } + cfg.LearnersNext = nil + + for id := range cfg.Voters[1] { + _, isVoter := cfg.Voters[0][id] + _, isLearner := cfg.Learners[id] + + if !isVoter && !isLearner { + delete(prs, id) + } + } + cfg.Voters[1] = nil + + return checkAndReturn(cfg, prs) +} + +// Simple carries out a series of configuration changes that (in aggregate) +// mutates the incoming majority config Voters[0] by at most one. This method +// will return an error if that is not the case, if the resulting quorum is +// zero, or if the configuration is in a joint state (i.e. if there is an +// outgoing configuration). +func (c Changer) Simple(ccs ...pb.ConfChange) (tracker.Config, tracker.ProgressMap, error) { + cfg, prs, err := c.checkAndCopy() + if err != nil { + return c.err(err) + } + if joint(cfg) { + err := errors.New("can't apply simple config change in joint config") + return c.err(err) + } + if err := c.apply(&cfg, prs, ccs...); err != nil { + return c.err(err) + } + if n := symdiff(c.Tracker.Voters[0], cfg.Voters[0]); n > 1 { + return tracker.Config{}, nil, errors.New("more than voter changed without entering joint config") + } + if err := checkInvariants(cfg, prs); err != nil { + return tracker.Config{}, tracker.ProgressMap{}, nil + } + + return checkAndReturn(cfg, prs) +} + +// apply a ConfChange to the configuration. By convention, changes to voters are +// always made to the incoming majority config Voters[0]. Voters[1] is either +// empty or preserves the outgoing majority configuration while in a joint state. +func (c Changer) apply(cfg *tracker.Config, prs tracker.ProgressMap, ccs ...pb.ConfChange) error { + for _, cc := range ccs { + if cc.NodeID == 0 { + // etcd replaces the NodeID with zero if it decides (downstream of + // raft) to not apply a ConfChange, so we have to have explicit code + // here to ignore these. + continue + } + switch cc.Type { + case pb.ConfChangeAddNode: + c.makeVoter(cfg, prs, cc.NodeID) + case pb.ConfChangeAddLearnerNode: + c.makeLearner(cfg, prs, cc.NodeID) + case pb.ConfChangeRemoveNode: + c.remove(cfg, prs, cc.NodeID) + case pb.ConfChangeUpdateNode: + default: + return fmt.Errorf("unexpected conf type %d", cc.Type) + } + } + if len(cfg.Voters[0]) == 0 { + return errors.New("removed all voters") + } + return nil +} + +// makeVoter adds or promotes the given ID to be a voter in the incoming +// majority config. +func (c Changer) makeVoter(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + pr := prs[id] + if pr == nil { + c.initProgress(cfg, prs, id, false /* isLearner */) + return + } + + pr.IsLearner = false + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + cfg.Voters[0][id] = struct{}{} + return +} + +// makeLearner makes the given ID a learner or stages it to be a learner once +// an active joint configuration is exited. +// +// The former happens when the peer is not a part of the outgoing config, in +// which case we either add a new learner or demote a voter in the incoming +// config. +// +// The latter case occurs when the configuration is joint and the peer is a +// voter in the outgoing config. In that case, we do not want to add the peer +// as a learner because then we'd have to track a peer as a voter and learner +// simultaneously. Instead, we add the learner to LearnersNext, so that it will +// be added to Learners the moment the outgoing config is removed by +// LeaveJoint(). +func (c Changer) makeLearner(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + pr := prs[id] + if pr == nil { + c.initProgress(cfg, prs, id, true /* isLearner */) + return + } + if pr.IsLearner { + return + } + // Remove any existing voter in the incoming config... + c.remove(cfg, prs, id) + // ... but save the Progress. + prs[id] = pr + // Use LearnersNext if we can't add the learner to Learners directly, i.e. + // if the peer is still tracked as a voter in the outgoing config. It will + // be turned into a learner in LeaveJoint(). + // + // Otherwise, add a regular learner right away. + if _, onRight := cfg.Voters[1][id]; onRight { + nilAwareAdd(&cfg.LearnersNext, id) + } else { + pr.IsLearner = true + nilAwareAdd(&cfg.Learners, id) + } +} + +// remove this peer as a voter or learner from the incoming config. +func (c Changer) remove(cfg *tracker.Config, prs tracker.ProgressMap, id uint64) { + if _, ok := prs[id]; !ok { + return + } + + delete(cfg.Voters[0], id) + nilAwareDelete(&cfg.Learners, id) + nilAwareDelete(&cfg.LearnersNext, id) + + // If the peer is still a voter in the outgoing config, keep the Progress. + if _, onRight := cfg.Voters[1][id]; !onRight { + delete(prs, id) + } +} + +// initProgress initializes a new progress for the given node or learner. +func (c Changer) initProgress(cfg *tracker.Config, prs tracker.ProgressMap, id uint64, isLearner bool) { + if !isLearner { + cfg.Voters[0][id] = struct{}{} + } else { + nilAwareAdd(&cfg.Learners, id) + } + prs[id] = &tracker.Progress{ + // We initialize Progress.Next with lastIndex+1 so that the peer will be + // probed without an index first. + // + // TODO(tbg): verify that, this is just my best guess. + Next: c.LastIndex + 1, + Match: 0, + Inflights: tracker.NewInflights(c.Tracker.MaxInflight), + IsLearner: isLearner, + // When a node is first added, we should mark it as recently active. + // Otherwise, CheckQuorum may cause us to step down if it is invoked + // before the added node has had a chance to communicate with us. + RecentActive: true, + } +} + +// checkInvariants makes sure that the config and progress are compatible with +// each other. This is used to check both what the Changer is initialized with, +// as well as what it returns. +func checkInvariants(cfg tracker.Config, prs tracker.ProgressMap) error { + // NB: intentionally allow the empty config. In production we'll never see a + // non-empty config (we prevent it from being created) but we will need to + // be able to *create* an initial config, for example during bootstrap (or + // during tests). Instead of having to hand-code this, we allow + // transitioning from an empty config into any other legal and non-empty + // config. + for _, ids := range []map[uint64]struct{}{ + cfg.Voters.IDs(), + cfg.Learners, + cfg.LearnersNext, + } { + for id := range ids { + if _, ok := prs[id]; !ok { + return fmt.Errorf("no progress for %d", id) + } + } + } + + // Any staged learner was staged because it could not be directly added due + // to a conflicting voter in the outgoing config. + for id := range cfg.LearnersNext { + if _, ok := cfg.Voters[1][id]; !ok { + return fmt.Errorf("%d is in LearnersNext, but not Voters[1]", id) + } + if prs[id].IsLearner { + return fmt.Errorf("%d is in LearnersNext, but is already marked as learner", id) + } + } + // Conversely Learners and Voters doesn't intersect at all. + for id := range cfg.Learners { + if _, ok := cfg.Voters[1][id]; ok { + return fmt.Errorf("%d is in Learners and Voters[1]", id) + } + if _, ok := cfg.Voters[0][id]; ok { + return fmt.Errorf("%d is in Learners and Voters[0]", id) + } + if !prs[id].IsLearner { + return fmt.Errorf("%d is in Learners, but is not marked as learner", id) + } + } + + if !joint(cfg) { + // We enforce that empty maps are nil instead of zero. + if cfg.Voters[1] != nil { + return fmt.Errorf("Voters[1] must be nil when not joint") + } + if cfg.LearnersNext != nil { + return fmt.Errorf("LearnersNext must be nil when not joint") + } + } + + return nil +} + +// checkAndCopy copies the tracker's config and progress map (deeply enough for +// the purposes of the Changer) and returns those copies. It returns an error +// if checkInvariants does. +func (c Changer) checkAndCopy() (tracker.Config, tracker.ProgressMap, error) { + cfg := c.Tracker.Config.Clone() + prs := tracker.ProgressMap{} + + for id, pr := range c.Tracker.Progress { + // A shallow copy is enough because we only mutate the Learner field. + ppr := *pr + prs[id] = &ppr + } + return checkAndReturn(cfg, prs) +} + +// checkAndReturn calls checkInvariants on the input and returns either the +// resulting error or the input. +func checkAndReturn(cfg tracker.Config, prs tracker.ProgressMap) (tracker.Config, tracker.ProgressMap, error) { + if err := checkInvariants(cfg, prs); err != nil { + return tracker.Config{}, tracker.ProgressMap{}, err + } + return cfg, prs, nil +} + +// err returns zero values and an error. +func (c Changer) err(err error) (tracker.Config, tracker.ProgressMap, error) { + return tracker.Config{}, nil, err +} + +// nilAwareAdd populates a map entry, creating the map if necessary. +func nilAwareAdd(m *map[uint64]struct{}, id uint64) { + if *m == nil { + *m = map[uint64]struct{}{} + } + (*m)[id] = struct{}{} +} + +// nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after. +func nilAwareDelete(m *map[uint64]struct{}, id uint64) { + if *m == nil { + return + } + delete(*m, id) + if len(*m) == 0 { + *m = nil + } +} + +// symdiff returns the count of the symmetric difference between the sets of +// uint64s, i.e. len( (l - r) \union (r - l)). +func symdiff(l, r map[uint64]struct{}) int { + var n int + pairs := [][2]quorum.MajorityConfig{ + {l, r}, // count elems in l but not in r + {r, l}, // count elems in r but not in l + } + for _, p := range pairs { + for id := range p[0] { + if _, ok := p[1][id]; !ok { + n++ + } + } + } + return n +} + +func joint(cfg tracker.Config) bool { + return len(cfg.Voters[1]) > 0 +} + +// Describe prints the type and NodeID of the configuration changes as a +// space-delimited string. +func Describe(ccs ...pb.ConfChange) string { + var buf strings.Builder + for _, cc := range ccs { + if buf.Len() > 0 { + buf.WriteByte(' ') + } + fmt.Fprintf(&buf, "%s(%d)", cc.Type, cc.NodeID) + } + return buf.String() +} diff --git a/raft/confchange/datadriven_test.go b/raft/confchange/datadriven_test.go new file mode 100644 index 000000000000..7d5428f17906 --- /dev/null +++ b/raft/confchange/datadriven_test.go @@ -0,0 +1,105 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/cockroachdb/datadriven" + pb "go.etcd.io/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/tracker" +) + +func TestConfChangeDataDriven(t *testing.T) { + datadriven.Walk(t, "testdata", func(t *testing.T, path string) { + tr := tracker.MakeProgressTracker(10) + c := Changer{ + Tracker: tr, + LastIndex: 0, // incremented in this test with each cmd + } + + // The test files use the commands + // - simple: run a simple conf change (i.e. no joint consensus), + // - enter-joint: enter a joint config, and + // - leave-joint: leave a joint config. + // The first two take a list of config changes, which have the following + // syntax: + // - vn: make n a voter, + // - ln: make n a learner, + // - rn: remove n, and + // - un: update n. + datadriven.RunTest(t, path, func(d *datadriven.TestData) string { + defer func() { + c.LastIndex++ + }() + var ccs []pb.ConfChange + toks := strings.Split(strings.TrimSpace(d.Input), " ") + if toks[0] == "" { + toks = nil + } + for _, tok := range toks { + if len(tok) < 2 { + return fmt.Sprintf("unknown token %s", tok) + } + var cc pb.ConfChange + switch tok[0] { + case 'v': + cc.Type = pb.ConfChangeAddNode + case 'l': + cc.Type = pb.ConfChangeAddLearnerNode + case 'r': + cc.Type = pb.ConfChangeRemoveNode + case 'u': + cc.Type = pb.ConfChangeUpdateNode + default: + return fmt.Sprintf("unknown input: %s", tok) + } + id, err := strconv.ParseUint(tok[1:], 10, 64) + if err != nil { + return err.Error() + } + cc.NodeID = id + ccs = append(ccs, cc) + } + + var cfg tracker.Config + var prs tracker.ProgressMap + var err error + switch d.Cmd { + case "simple": + cfg, prs, err = c.Simple(ccs...) + case "enter-joint": + cfg, prs, err = c.EnterJoint(ccs...) + case "leave-joint": + if len(ccs) > 0 { + err = errors.New("this command takes no input") + } else { + cfg, prs, err = c.LeaveJoint() + } + default: + return "unknown command" + } + if err != nil { + return err.Error() + "\n" + } + c.Tracker.Config, c.Tracker.Progress = cfg, prs + return fmt.Sprintf("%s\n%s", c.Tracker.Config, c.Tracker.Progress) + }) + }) +} diff --git a/raft/confchange/quick_test.go b/raft/confchange/quick_test.go new file mode 100644 index 000000000000..66ac37d8fc8c --- /dev/null +++ b/raft/confchange/quick_test.go @@ -0,0 +1,169 @@ +// Copyright 2019 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package confchange + +import ( + "math/rand" + "reflect" + "testing" + "testing/quick" + + "github.com/motomux/pretty" + pb "go.etcd.io/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/tracker" +) + +// TestConfChangeQuick uses quickcheck to verify that simple and joint config +// changes arrive at the same result. +func TestConfChangeQuick(t *testing.T) { + cfg := &quick.Config{ + MaxCount: 1000, + } + + // Log the first couple of runs to give some indication of things working + // as intended. + const infoCount = 5 + + runWithJoint := func(c *Changer, ccs []pb.ConfChange) error { + cfg, prs, err := c.EnterJoint(ccs...) + if err != nil { + return err + } + c.Tracker.Config = cfg + c.Tracker.Progress = prs + cfg, prs, err = c.LeaveJoint() + if err != nil { + return err + } + c.Tracker.Config = cfg + c.Tracker.Progress = prs + return nil + } + + runWithSimple := func(c *Changer, ccs []pb.ConfChange) error { + for _, cc := range ccs { + cfg, prs, err := c.Simple(cc) + if err != nil { + return err + } + c.Tracker.Config, c.Tracker.Progress = cfg, prs + } + return nil + } + + type testFunc func(*Changer, []pb.ConfChange) error + + wrapper := func(invoke testFunc) func(setup initialChanges, ccs confChanges) (*Changer, error) { + return func(setup initialChanges, ccs confChanges) (*Changer, error) { + tr := tracker.MakeProgressTracker(10) + c := &Changer{ + Tracker: tr, + LastIndex: 10, + } + + if err := runWithSimple(c, setup); err != nil { + return nil, err + } + + err := invoke(c, ccs) + return c, err + } + } + + var n int + f1 := func(setup initialChanges, ccs confChanges) *Changer { + c, err := wrapper(runWithSimple)(setup, ccs) + if err != nil { + t.Fatal(err) + } + if n < infoCount { + t.Log("initial setup:", Describe(setup...)) + t.Log("changes:", Describe(ccs...)) + t.Log(c.Tracker.Config) + t.Log(c.Tracker.Progress) + } + n++ + return c + } + f2 := func(setup initialChanges, ccs confChanges) *Changer { + c, err := wrapper(runWithJoint)(setup, ccs) + if err != nil { + t.Fatal(err) + } + return c + } + err := quick.CheckEqual(f1, f2, cfg) + if err == nil { + return + } + cErr, ok := err.(*quick.CheckEqualError) + if !ok { + t.Fatal(err) + } + + t.Error("setup:", Describe(cErr.In[0].([]pb.ConfChange)...)) + t.Error("ccs:", Describe(cErr.In[1].([]pb.ConfChange)...)) + pretty.Ldiff(t, cErr.Out1, cErr.Out2) +} + +type confChangeTyp pb.ConfChangeType + +func (confChangeTyp) Generate(rand *rand.Rand, _ int) reflect.Value { + return reflect.ValueOf(confChangeTyp(rand.Intn(4))) +} + +type confChanges []pb.ConfChange + +func genCC(num func() int, id func() uint64, typ func() pb.ConfChangeType) []pb.ConfChange { + var ccs []pb.ConfChange + n := num() + for i := 0; i < n; i++ { + ccs = append(ccs, pb.ConfChange{Type: typ(), NodeID: id()}) + } + return ccs +} + +func (confChanges) Generate(rand *rand.Rand, _ int) reflect.Value { + num := func() int { + return 1 + rand.Intn(9) + } + id := func() uint64 { + // Note that num() >= 1, so we're never returning 1 from this method, + // meaning that we'll never touch NodeID one, which is special to avoid + // voterless configs altogether in this test. + return 1 + uint64(num()) + } + typ := func() pb.ConfChangeType { + return pb.ConfChangeType(rand.Intn(len(pb.ConfChangeType_name))) + } + return reflect.ValueOf(genCC(num, id, typ)) +} + +type initialChanges []pb.ConfChange + +func (initialChanges) Generate(rand *rand.Rand, _ int) reflect.Value { + num := func() int { + return 1 + rand.Intn(5) + } + id := func() uint64 { return uint64(num()) } + typ := func() pb.ConfChangeType { + return pb.ConfChangeAddNode + } + // NodeID one is special - it's in the initial config and will be a voter + // always (this is to avoid uninteresting edge cases where the simple conf + // changes can't easily make progress). + ccs := append([]pb.ConfChange{{Type: pb.ConfChangeAddNode, NodeID: 1}}, genCC(num, id, typ)...) + return reflect.ValueOf(ccs) +} diff --git a/raft/confchange/testdata/joint_idempotency.txt b/raft/confchange/testdata/joint_idempotency.txt new file mode 100644 index 000000000000..2bd3144fd36a --- /dev/null +++ b/raft/confchange/testdata/joint_idempotency.txt @@ -0,0 +1,23 @@ +# Verify that operations upon entering the joint state are idempotent, i.e. +# removing an absent node is fine, etc. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +enter-joint +r1 r2 r9 v2 v3 v4 v2 v3 v4 l2 l2 r4 r4 l1 l1 +---- +voters=(3)&&(1) learners=(2) learners_next=(1) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 learner +3: StateProbe match=0 next=2 + +leave-joint +---- +voters=(3) learners=(1 2) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=2 learner +3: StateProbe match=0 next=2 diff --git a/raft/confchange/testdata/joint_learners_next.txt b/raft/confchange/testdata/joint_learners_next.txt new file mode 100644 index 000000000000..e3ddf4cd3f29 --- /dev/null +++ b/raft/confchange/testdata/joint_learners_next.txt @@ -0,0 +1,24 @@ +# Verify that when a voter is demoted in a joint config, it will show up in +# learners_next until the joint config is left, and only then will the progress +# turn into that of a learner, without resetting the progress. Note that this +# last fact is verified by `next`, which can tell us which "round" the progress +# was originally created in. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +enter-joint +v2 l1 +---- +voters=(2)&&(1) learners_next=(1) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 + +leave-joint +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=2 diff --git a/raft/confchange/testdata/joint_safety.txt b/raft/confchange/testdata/joint_safety.txt new file mode 100644 index 000000000000..f123b60ec7e5 --- /dev/null +++ b/raft/confchange/testdata/joint_safety.txt @@ -0,0 +1,81 @@ +leave-joint +---- +can't leave a non-joint config + +enter-joint +---- +can't make a zero-voter config joint + +enter-joint +v1 +---- +can't make a zero-voter config joint + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=4 + +leave-joint +---- +can't leave a non-joint config + +# Can enter into joint config. +enter-joint +---- +voters=(1)&&(1) +1: StateProbe match=0 next=4 + +enter-joint +---- +config is already joint + +leave-joint +---- +voters=(1) +1: StateProbe match=0 next=4 + +leave-joint +---- +can't leave a non-joint config + +# Can enter again, this time with some ops. +enter-joint +r1 v2 v3 l4 +---- +voters=(2 3)&&(1) learners=(4) +1: StateProbe match=0 next=4 +2: StateProbe match=0 next=10 +3: StateProbe match=0 next=10 +4: StateProbe match=0 next=10 learner + +enter-joint +---- +config is already joint + +enter-joint +v12 +---- +config is already joint + +simple +l15 +---- +can't apply simple config change in joint config + +leave-joint +---- +voters=(2 3) learners=(4) +2: StateProbe match=0 next=10 +3: StateProbe match=0 next=10 +4: StateProbe match=0 next=10 learner + +simple +l9 +---- +voters=(2 3) learners=(4 9) +2: StateProbe match=0 next=10 +3: StateProbe match=0 next=10 +4: StateProbe match=0 next=10 learner +9: StateProbe match=0 next=15 learner diff --git a/raft/confchange/testdata/simple_idempotency.txt b/raft/confchange/testdata/simple_idempotency.txt new file mode 100644 index 000000000000..a663a88a3f72 --- /dev/null +++ b/raft/confchange/testdata/simple_idempotency.txt @@ -0,0 +1,69 @@ +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=3 + +simple +l1 +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=3 + +simple +l1 +---- +voters=(2) learners=(1) +1: StateProbe match=0 next=1 learner +2: StateProbe match=0 next=3 + +simple +r1 +---- +voters=(2) +2: StateProbe match=0 next=3 + +simple +r1 +---- +voters=(2) +2: StateProbe match=0 next=3 + +simple +v3 +---- +voters=(2 3) +2: StateProbe match=0 next=3 +3: StateProbe match=0 next=8 + +simple +r3 +---- +voters=(2) +2: StateProbe match=0 next=3 + +simple +r3 +---- +voters=(2) +2: StateProbe match=0 next=3 + +simple +r4 +---- +voters=(2) +2: StateProbe match=0 next=3 diff --git a/raft/confchange/testdata/simple_promote_demote.txt b/raft/confchange/testdata/simple_promote_demote.txt new file mode 100644 index 000000000000..651bc157381e --- /dev/null +++ b/raft/confchange/testdata/simple_promote_demote.txt @@ -0,0 +1,60 @@ +# Set up three voters for this test. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 + +simple +v3 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 +3: StateProbe match=0 next=3 + +# Can atomically demote and promote without a hitch. +# This is pointless, but possible. +simple +l1 v1 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 +3: StateProbe match=0 next=3 + +# Can demote a voter. +simple +l2 +---- +voters=(1 3) learners=(2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 learner +3: StateProbe match=0 next=3 + +# Can atomically promote and demote the same voter. +# This is pointless, but possible. +simple +v2 l2 +---- +voters=(1 3) learners=(2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 learner +3: StateProbe match=0 next=3 + +# Can promote a voter. +simple +v2 +---- +voters=(1 2 3) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 +3: StateProbe match=0 next=3 diff --git a/raft/confchange/testdata/simple_safety.txt b/raft/confchange/testdata/simple_safety.txt new file mode 100644 index 000000000000..4bf420fc14b2 --- /dev/null +++ b/raft/confchange/testdata/simple_safety.txt @@ -0,0 +1,64 @@ +simple +l1 +---- +removed all voters + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=2 + +simple +v2 l3 +---- +voters=(1 2) learners=(3) +1: StateProbe match=0 next=2 +2: StateProbe match=0 next=3 +3: StateProbe match=0 next=3 learner + +simple +r1 v5 +---- +more than voter changed without entering joint config + +simple +r1 r2 +---- +removed all voters + +simple +v3 v4 +---- +more than voter changed without entering joint config + +simple +l1 v5 +---- +more than voter changed without entering joint config + +simple +l1 l2 +---- +removed all voters + +simple +l2 l3 l4 l5 +---- +voters=(1) learners=(2 3 4 5) +1: StateProbe match=0 next=2 +2: StateProbe match=0 next=3 learner +3: StateProbe match=0 next=3 learner +4: StateProbe match=0 next=9 learner +5: StateProbe match=0 next=9 learner + +simple +r1 +---- +removed all voters + +simple +r2 r3 r4 r5 +---- +voters=(1) +1: StateProbe match=0 next=2 diff --git a/raft/confchange/testdata/update.txt b/raft/confchange/testdata/update.txt new file mode 100644 index 000000000000..ffc2922c9cbd --- /dev/null +++ b/raft/confchange/testdata/update.txt @@ -0,0 +1,23 @@ +# Nobody cares about ConfChangeUpdateNode, but at least use it once. It is used +# by etcd as a convenient way to pass a blob through their conf change machinery +# that updates information tracked outside of raft. + +simple +v1 +---- +voters=(1) +1: StateProbe match=0 next=1 + +simple +v2 u1 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 + +simple +u1 u2 u3 u1 u2 u3 +---- +voters=(1 2) +1: StateProbe match=0 next=1 +2: StateProbe match=0 next=2 diff --git a/raft/confchange/testdata/zero.txt b/raft/confchange/testdata/zero.txt new file mode 100644 index 000000000000..226ade088323 --- /dev/null +++ b/raft/confchange/testdata/zero.txt @@ -0,0 +1,6 @@ +# NodeID zero is ignored. +simple +v1 r0 v0 l0 +---- +voters=(1) +1: StateProbe match=0 next=1 diff --git a/raft/raft.go b/raft/raft.go index 846ff496ffc6..01e23ec98942 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "go.etcd.io/etcd/raft/confchange" "go.etcd.io/etcd/raft/quorum" pb "go.etcd.io/etcd/raft/raftpb" "go.etcd.io/etcd/raft/tracker" @@ -356,15 +357,11 @@ func newRaft(c *Config) *raft { } for _, p := range peers { // Add node to active config. - r.prs.InitProgress(p, 0 /* match */, 1 /* next */, false /* isLearner */) + r.applyConfChange(pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: p}) } for _, p := range learners { // Add learner to active config. - r.prs.InitProgress(p, 0 /* match */, 1 /* next */, true /* isLearner */) - - if r.id == p { - r.isLearner = true - } + r.applyConfChange(pb.ConfChange{Type: pb.ConfChangeAddLearnerNode, NodeID: p}) } if !isHardStateEqual(hs, emptyState) { @@ -1401,55 +1398,15 @@ func (r *raft) promotable() bool { } func (r *raft) applyConfChange(cc pb.ConfChange) pb.ConfState { - addNodeOrLearnerNode := func(id uint64, isLearner bool) { - // NB: this method is intentionally hidden from view. All mutations of - // the conf state must call applyConfChange directly. - pr := r.prs.Progress[id] - if pr == nil { - r.prs.InitProgress(id, 0, r.raftLog.lastIndex()+1, isLearner) - } else { - if isLearner && !pr.IsLearner { - // Can only change Learner to Voter. - // - // TODO(tbg): why? - r.logger.Infof("%x ignored addLearner: do not support changing %x from raft peer to learner.", r.id, id) - return - } - - if isLearner == pr.IsLearner { - // Ignore any redundant addNode calls (which can happen because the - // initial bootstrapping entries are applied twice). - return - } - - // Change Learner to Voter, use origin Learner progress. - r.prs.RemoveAny(id) - r.prs.InitProgress(id, 0 /* match */, 1 /* next */, false /* isLearner */) - pr.IsLearner = false - *r.prs.Progress[id] = *pr - } - - // When a node is first added, we should mark it as recently active. - // Otherwise, CheckQuorum may cause us to step down if it is invoked - // before the added node has had a chance to communicate with us. - r.prs.Progress[id].RecentActive = true - } - - var removed int - if cc.NodeID != None { - switch cc.Type { - case pb.ConfChangeAddNode: - addNodeOrLearnerNode(cc.NodeID, false /* isLearner */) - case pb.ConfChangeAddLearnerNode: - addNodeOrLearnerNode(cc.NodeID, true /* isLearner */) - case pb.ConfChangeRemoveNode: - removed++ - r.prs.RemoveAny(cc.NodeID) - case pb.ConfChangeUpdateNode: - default: - panic("unexpected conf type") - } + cfg, prs, err := confchange.Changer{ + Tracker: r.prs, + LastIndex: r.raftLog.lastIndex(), + }.Simple(cc) + if err != nil { + panic(err) } + r.prs.Config = cfg + r.prs.Progress = prs r.logger.Infof("%x switched to configuration %s", r.id, r.prs.Config) // Now that the configuration is updated, handle any side effects. @@ -1479,12 +1436,10 @@ func (r *raft) applyConfChange(cc pb.ConfChange) pb.ConfState { if r.state != StateLeader || len(cs.Nodes) == 0 { return cs } - if removed > 0 { + if r.maybeCommit() { // The quorum size may have been reduced (but not to zero), so see if // any pending entries can be committed. - if r.maybeCommit() { - r.bcastAppend() - } + r.bcastAppend() } // If the the leadTransferee was removed, abort the leadership transfer. if _, tOK := r.prs.Progress[r.leadTransferee]; !tOK && r.leadTransferee != 0 { diff --git a/raft/raft_test.go b/raft/raft_test.go index 6d9a26efd895..fc27fee11ea0 100644 --- a/raft/raft_test.go +++ b/raft/raft_test.go @@ -1140,9 +1140,13 @@ func TestCommit(t *testing.T) { storage.hardState = pb.HardState{Term: tt.smTerm} sm := newTestRaft(1, []uint64{1}, 10, 2, storage) - sm.prs.RemoveAny(1) for j := 0; j < len(tt.matches); j++ { - sm.prs.InitProgress(uint64(j)+1, tt.matches[j], tt.matches[j]+1, false) + id := uint64(j) + 1 + if id > 1 { + sm.applyConfChange(pb.ConfChange{Type: pb.ConfChangeAddNode, NodeID: id}) + } + pr := sm.prs.Progress[id] + pr.Match, pr.Next = tt.matches[j], tt.matches[j]+1 } sm.maybeCommit() if g := sm.raftLog.committed; g != tt.w { @@ -1927,7 +1931,7 @@ func TestNonPromotableVoterWithCheckQuorum(t *testing.T) { nt := newNetwork(a, b) setRandomizedElectionTimeout(b, b.electionTimeout+1) // Need to remove 2 again to make it a non-promotable node since newNetwork overwritten some internal states - b.prs.RemoveAny(2) + b.applyConfChange(pb.ConfChange{Type: pb.ConfChangeRemoveNode, NodeID: 2}) if b.promotable() { t.Fatalf("promotable = %v, want false", b.promotable()) @@ -3093,14 +3097,42 @@ func TestAddNode(t *testing.T) { // TestAddLearner tests that addLearner could update nodes correctly. func TestAddLearner(t *testing.T) { r := newTestRaft(1, []uint64{1}, 10, 1, NewMemoryStorage()) + // Add new learner peer. r.applyConfChange(pb.ConfChange{NodeID: 2, Type: pb.ConfChangeAddLearnerNode}) + if r.isLearner { + t.Fatal("expected 1 to be voter") + } nodes := r.prs.LearnerNodes() wnodes := []uint64{2} if !reflect.DeepEqual(nodes, wnodes) { t.Errorf("nodes = %v, want %v", nodes, wnodes) } if !r.prs.Progress[2].IsLearner { - t.Errorf("node 2 is learner %t, want %t", r.prs.Progress[2].IsLearner, true) + t.Fatal("expected 2 to be learner") + } + + // Promote peer to voter. + r.applyConfChange(pb.ConfChange{NodeID: 2, Type: pb.ConfChangeAddNode}) + if r.prs.Progress[2].IsLearner { + t.Fatal("expected 2 to be voter") + } + + // Demote r. + r.applyConfChange(pb.ConfChange{NodeID: 1, Type: pb.ConfChangeAddLearnerNode}) + if !r.prs.Progress[1].IsLearner { + t.Fatal("expected 1 to be learner") + } + if !r.isLearner { + t.Fatal("expected 1 to be learner") + } + + // Promote r again. + r.applyConfChange(pb.ConfChange{NodeID: 1, Type: pb.ConfChangeAddNode}) + if r.prs.Progress[1].IsLearner { + t.Fatal("expected 1 to be voter") + } + if r.isLearner { + t.Fatal("expected 1 to be voter") } } @@ -3148,12 +3180,13 @@ func TestRemoveNode(t *testing.T) { t.Errorf("nodes = %v, want %v", g, w) } - // remove all nodes from cluster + // Removing the remaining voter will panic. + defer func() { + if r := recover(); r == nil { + t.Error("did not panic") + } + }() r.applyConfChange(pb.ConfChange{NodeID: 1, Type: pb.ConfChangeRemoveNode}) - w = []uint64{} - if g := r.prs.VoterNodes(); !reflect.DeepEqual(g, w) { - t.Errorf("nodes = %v, want %v", g, w) - } } // TestRemoveLearner tests that removeNode could update nodes and @@ -3171,12 +3204,15 @@ func TestRemoveLearner(t *testing.T) { t.Errorf("nodes = %v, want %v", g, w) } - // remove all nodes from cluster + // Removing the remaining voter will panic. + defer func() { + if r := recover(); r == nil { + t.Error("did not panic") + } + }() r.applyConfChange(pb.ConfChange{NodeID: 1, Type: pb.ConfChangeRemoveNode}) - if g := r.prs.VoterNodes(); !reflect.DeepEqual(g, w) { - t.Errorf("nodes = %v, want %v", g, w) - } } + func TestPromotable(t *testing.T) { id := uint64(1) tests := []struct { @@ -4124,12 +4160,16 @@ func newNetworkWithConfig(configFunc func(*Config), peers ...stateMachine) *netw sm := newRaft(cfg) npeers[id] = sm case *raft: + // TODO(tbg): this is all pretty confused. Clean this up. learners := make(map[uint64]bool, len(v.prs.Learners)) for i := range v.prs.Learners { learners[i] = true } v.id = id v.prs = tracker.MakeProgressTracker(v.prs.MaxInflight) + if len(learners) > 0 { + v.prs.Learners = map[uint64]struct{}{} + } for i := 0; i < size; i++ { pr := &tracker.Progress{} if _, ok := learners[peerAddrs[i]]; ok { diff --git a/raft/tracker/progress.go b/raft/tracker/progress.go index a7f1ab7d38fc..697277b26430 100644 --- a/raft/tracker/progress.go +++ b/raft/tracker/progress.go @@ -16,6 +16,7 @@ package tracker import ( "fmt" + "sort" "strings" ) @@ -235,3 +236,22 @@ func (pr *Progress) String() string { } return buf.String() } + +// ProgressMap is a map of *Progress. +type ProgressMap map[uint64]*Progress + +// String prints the ProgressMap in sorted key order, one Progress per line. +func (m ProgressMap) String() string { + ids := make([]uint64, 0, len(m)) + for k := range m { + ids = append(ids, k) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + var buf strings.Builder + for _, id := range ids { + fmt.Fprintf(&buf, "%d: %s\n", id, m[id]) + } + return buf.String() +} diff --git a/raft/tracker/tracker.go b/raft/tracker/tracker.go index 4b3396fbe170..a2638f5f01e5 100644 --- a/raft/tracker/tracker.go +++ b/raft/tracker/tracker.go @@ -17,6 +17,7 @@ package tracker import ( "fmt" "sort" + "strings" "go.etcd.io/etcd/raft/quorum" ) @@ -33,12 +34,11 @@ type Config struct { // simplifies the implementation since it allows peers to have clarity about // its current role without taking into account joint consensus. Learners map[uint64]struct{} - // TODO(tbg): when we actually carry out joint consensus changes and turn a - // voter into a learner, we cannot add the learner when entering the joint - // state. This is because this would violate the invariant that the inter- - // section of voters and learners is empty. For example, assume a Voter is - // removed and immediately re-added as a learner (or in other words, it is - // demoted). + // When we turn a voter into a learner during a joint consensus transition, + // we cannot add the learner directly when entering the joint state. This is + // because this would violate the invariant that the intersection of + // voters and learners is empty. For example, assume a Voter is removed and + // immediately re-added as a learner (or in other words, it is demoted): // // Initially, the configuration will be // @@ -51,7 +51,7 @@ type Config struct { // learners: {3} // // but this violates the invariant (3 is both voter and learner). Instead, - // we have + // we get // // voters: {1 2} & {1 2 3} // learners: {} @@ -66,20 +66,40 @@ type Config struct { // // Note that next_learners is not used while adding a learner that is not // also a voter in the joint config. In this case, the learner is added - // to Learners right away when entering the joint configuration, so that it - // is caught up as soon as possible. - // - // NextLearners map[uint64]struct{} + // right away when entering the joint configuration, so that it is caught up + // as soon as possible. + LearnersNext map[uint64]struct{} +} + +func (c Config) String() string { + var buf strings.Builder + fmt.Fprintf(&buf, "voters=%s", c.Voters) + if c.Learners != nil { + fmt.Fprintf(&buf, " learners=%s", quorum.MajorityConfig(c.Learners).String()) + } + if c.LearnersNext != nil { + fmt.Fprintf(&buf, " learners_next=%s", quorum.MajorityConfig(c.LearnersNext).String()) + } + return buf.String() } -func (c *Config) String() string { - if len(c.Learners) == 0 { - return fmt.Sprintf("voters=%s", c.Voters) +// Clone returns a copy of the Config that shares no memory with the original. +func (c *Config) Clone() Config { + clone := func(m map[uint64]struct{}) map[uint64]struct{} { + if m == nil { + return nil + } + mm := make(map[uint64]struct{}, len(m)) + for k := range m { + mm[k] = struct{}{} + } + return mm + } + return Config{ + Voters: quorum.JointConfig{clone(c.Voters[0]), clone(c.Voters[1])}, + Learners: clone(c.Learners), + LearnersNext: clone(c.LearnersNext), } - return fmt.Sprintf( - "voters=%s learners=%s", - c.Voters, quorum.MajorityConfig(c.Learners).String(), - ) } // ProgressTracker tracks the currently active configuration and the information @@ -88,7 +108,7 @@ func (c *Config) String() string { type ProgressTracker struct { Config - Progress map[uint64]*Progress + Progress ProgressMap Votes map[uint64]bool @@ -102,11 +122,10 @@ func MakeProgressTracker(maxInflight int) ProgressTracker { Config: Config{ Voters: quorum.JointConfig{ quorum.MajorityConfig{}, - // TODO(tbg): this will be mostly empty, so make it a nil pointer - // in the common case. - quorum.MajorityConfig{}, + nil, // only populated when used }, - Learners: map[uint64]struct{}{}, + Learners: nil, // only populated when used + LearnersNext: nil, // only populated when used }, Votes: map[uint64]bool{}, Progress: map[uint64]*Progress{}, @@ -139,44 +158,6 @@ func (p *ProgressTracker) Committed() uint64 { return uint64(p.Voters.CommittedIndex(matchAckIndexer(p.Progress))) } -// RemoveAny removes this peer, which *must* be tracked as a voter or learner, -// from the tracker. -func (p *ProgressTracker) RemoveAny(id uint64) { - _, okPR := p.Progress[id] - _, okV1 := p.Voters[0][id] - _, okV2 := p.Voters[1][id] - _, okL := p.Learners[id] - - okV := okV1 || okV2 - - if !okPR { - panic("attempting to remove unknown peer %x") - } else if !okV && !okL { - panic("attempting to remove unknown peer %x") - } else if okV && okL { - panic(fmt.Sprintf("peer %x is both voter and learner", id)) - } - - delete(p.Voters[0], id) - delete(p.Voters[1], id) - delete(p.Learners, id) - delete(p.Progress, id) -} - -// InitProgress initializes a new progress for the given node or learner. The -// node may not exist yet in either form or a panic will ensue. -func (p *ProgressTracker) InitProgress(id, match, next uint64, isLearner bool) { - if pr := p.Progress[id]; pr != nil { - panic(fmt.Sprintf("peer %x already tracked as node %v", id, pr)) - } - if !isLearner { - p.Voters[0][id] = struct{}{} - } else { - p.Learners[id] = struct{}{} - } - p.Progress[id] = &Progress{Next: next, Match: match, Inflights: NewInflights(p.MaxInflight), IsLearner: isLearner} -} - // Visit invokes the supplied closure for all tracked progresses. func (p *ProgressTracker) Visit(f func(id uint64, pr *Progress)) { for id, pr := range p.Progress { diff --git a/vendor/github.com/cockroachdb/datadriven/datadriven.go b/vendor/github.com/cockroachdb/datadriven/datadriven.go index 49e73ce380f6..0b1954ade387 100644 --- a/vendor/github.com/cockroachdb/datadriven/datadriven.go +++ b/vendor/github.com/cockroachdb/datadriven/datadriven.go @@ -93,36 +93,41 @@ func runTestInternal( r := newTestDataReader(t, sourceName, reader, rewrite) for r.Next(t) { - d := &r.data - actual := func() string { - defer func() { - if r := recover(); r != nil { - fmt.Printf("\npanic during %s:\n%s\n", d.Pos, d.Input) - panic(r) - } + t.Run("", func(t *testing.T) { + d := &r.data + actual := func() string { + defer func() { + if r := recover(); r != nil { + fmt.Printf("\npanic during %s:\n%s\n", d.Pos, d.Input) + panic(r) + } + }() + return f(d) }() - return f(d) - }() - if r.rewrite != nil { - r.emit("----") - if hasBlankLine(actual) { - r.emit("----") - r.rewrite.WriteString(actual) + if r.rewrite != nil { r.emit("----") - r.emit("----") - } else { - r.emit(actual) - } - } else if d.Expected != actual { - t.Fatalf("\n%s: %s\nexpected:\n%s\nfound:\n%s", d.Pos, d.Input, d.Expected, actual) - } else if testing.Verbose() { - input := d.Input - if input == "" { - input = "" + if hasBlankLine(actual) { + r.emit("----") + r.rewrite.WriteString(actual) + r.emit("----") + r.emit("----") + } else { + r.emit(actual) + } + } else if d.Expected != actual { + t.Fatalf("\n%s: %s\nexpected:\n%s\nfound:\n%s", d.Pos, d.Input, d.Expected, actual) + } else if testing.Verbose() { + input := d.Input + if input == "" { + input = "" + } + // TODO(tbg): it's awkward to reproduce the args, but it would be helpful. + fmt.Printf("\n%s:\n%s [%d args]\n%s\n----\n%s", d.Pos, d.Cmd, len(d.CmdArgs), input, actual) } - // TODO(tbg): it's awkward to reproduce the args, but it would be helpful. - fmt.Printf("\n%s:\n%s [%d args]\n%s\n----\n%s", d.Pos, d.Cmd, len(d.CmdArgs), input, actual) + }) + if t.Failed() { + t.FailNow() } } diff --git a/vendor/github.com/kr/pty/pty_darwin.go b/vendor/github.com/kr/pty/pty_darwin.go index 4f4d5ca26eee..6344b6b0efb6 100644 --- a/vendor/github.com/kr/pty/pty_darwin.go +++ b/vendor/github.com/kr/pty/pty_darwin.go @@ -8,23 +8,28 @@ import ( ) func open() (pty, tty *os.File, err error) { - p, err := os.OpenFile("/dev/ptmx", os.O_RDWR, 0) + pFD, err := syscall.Open("/dev/ptmx", syscall.O_RDWR|syscall.O_CLOEXEC, 0) if err != nil { return nil, nil, err } + p := os.NewFile(uintptr(pFD), "/dev/ptmx") + // In case of error after this point, make sure we close the ptmx fd. + defer func() { + if err != nil { + _ = p.Close() // Best effort. + } + }() sname, err := ptsname(p) if err != nil { return nil, nil, err } - err = grantpt(p) - if err != nil { + if err := grantpt(p); err != nil { return nil, nil, err } - err = unlockpt(p) - if err != nil { + if err := unlockpt(p); err != nil { return nil, nil, err } diff --git a/vendor/github.com/kr/pty/pty_dragonfly.go b/vendor/github.com/kr/pty/pty_dragonfly.go index 5431fb5aec97..b7d1f20f29e5 100644 --- a/vendor/github.com/kr/pty/pty_dragonfly.go +++ b/vendor/github.com/kr/pty/pty_dragonfly.go @@ -14,19 +14,23 @@ func open() (pty, tty *os.File, err error) { if err != nil { return nil, nil, err } + // In case of error after this point, make sure we close the ptmx fd. + defer func() { + if err != nil { + _ = p.Close() // Best effort. + } + }() sname, err := ptsname(p) if err != nil { return nil, nil, err } - err = grantpt(p) - if err != nil { + if err := grantpt(p); err != nil { return nil, nil, err } - err = unlockpt(p) - if err != nil { + if err := unlockpt(p); err != nil { return nil, nil, err } diff --git a/vendor/github.com/kr/pty/pty_freebsd.go b/vendor/github.com/kr/pty/pty_freebsd.go index b341babd054b..63b6d91337ae 100644 --- a/vendor/github.com/kr/pty/pty_freebsd.go +++ b/vendor/github.com/kr/pty/pty_freebsd.go @@ -7,22 +7,28 @@ import ( "unsafe" ) -func posix_openpt(oflag int) (fd int, err error) { +func posixOpenpt(oflag int) (fd int, err error) { r0, _, e1 := syscall.Syscall(syscall.SYS_POSIX_OPENPT, uintptr(oflag), 0, 0) fd = int(r0) if e1 != 0 { err = e1 } - return + return fd, err } func open() (pty, tty *os.File, err error) { - fd, err := posix_openpt(syscall.O_RDWR | syscall.O_CLOEXEC) + fd, err := posixOpenpt(syscall.O_RDWR | syscall.O_CLOEXEC) if err != nil { return nil, nil, err } - p := os.NewFile(uintptr(fd), "/dev/pts") + // In case of error after this point, make sure we close the pts fd. + defer func() { + if err != nil { + _ = p.Close() // Best effort. + } + }() + sname, err := ptsname(p) if err != nil { return nil, nil, err @@ -42,7 +48,7 @@ func isptmaster(fd uintptr) (bool, error) { var ( emptyFiodgnameArg fiodgnameArg - ioctl_FIODGNAME = _IOW('f', 120, unsafe.Sizeof(emptyFiodgnameArg)) + ioctlFIODGNAME = _IOW('f', 120, unsafe.Sizeof(emptyFiodgnameArg)) ) func ptsname(f *os.File) (string, error) { @@ -59,8 +65,7 @@ func ptsname(f *os.File) (string, error) { buf = make([]byte, n) arg = fiodgnameArg{Len: n, Buf: (*byte)(unsafe.Pointer(&buf[0]))} ) - err = ioctl(f.Fd(), ioctl_FIODGNAME, uintptr(unsafe.Pointer(&arg))) - if err != nil { + if err := ioctl(f.Fd(), ioctlFIODGNAME, uintptr(unsafe.Pointer(&arg))); err != nil { return "", err } diff --git a/vendor/github.com/kr/pty/pty_linux.go b/vendor/github.com/kr/pty/pty_linux.go index cb901a21e006..296dd212985f 100644 --- a/vendor/github.com/kr/pty/pty_linux.go +++ b/vendor/github.com/kr/pty/pty_linux.go @@ -12,14 +12,19 @@ func open() (pty, tty *os.File, err error) { if err != nil { return nil, nil, err } + // In case of error after this point, make sure we close the ptmx fd. + defer func() { + if err != nil { + _ = p.Close() // Best effort. + } + }() sname, err := ptsname(p) if err != nil { return nil, nil, err } - err = unlockpt(p) - if err != nil { + if err := unlockpt(p); err != nil { return nil, nil, err } diff --git a/vendor/github.com/kr/pty/pty_openbsd.go b/vendor/github.com/kr/pty/pty_openbsd.go new file mode 100644 index 000000000000..6e7aeae7c079 --- /dev/null +++ b/vendor/github.com/kr/pty/pty_openbsd.go @@ -0,0 +1,33 @@ +package pty + +import ( + "os" + "syscall" + "unsafe" +) + +func open() (pty, tty *os.File, err error) { + /* + * from ptm(4): + * The PTMGET command allocates a free pseudo terminal, changes its + * ownership to the caller, revokes the access privileges for all previous + * users, opens the file descriptors for the master and slave devices and + * returns them to the caller in struct ptmget. + */ + + p, err := os.OpenFile("/dev/ptm", os.O_RDWR|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + defer p.Close() + + var ptm ptmget + if err := ioctl(p.Fd(), uintptr(ioctl_PTMGET), uintptr(unsafe.Pointer(&ptm))); err != nil { + return nil, nil, err + } + + pty = os.NewFile(uintptr(ptm.Cfd), "/dev/ptm") + tty = os.NewFile(uintptr(ptm.Sfd), "/dev/ptm") + + return pty, tty, nil +} diff --git a/vendor/github.com/kr/pty/pty_unsupported.go b/vendor/github.com/kr/pty/pty_unsupported.go index bd3d1e7e0e6f..9a3e721bc42b 100644 --- a/vendor/github.com/kr/pty/pty_unsupported.go +++ b/vendor/github.com/kr/pty/pty_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux,!darwin,!freebsd,!dragonfly +// +build !linux,!darwin,!freebsd,!dragonfly,!openbsd package pty diff --git a/vendor/github.com/kr/pty/types_openbsd.go b/vendor/github.com/kr/pty/types_openbsd.go new file mode 100644 index 000000000000..47701b5f9e01 --- /dev/null +++ b/vendor/github.com/kr/pty/types_openbsd.go @@ -0,0 +1,14 @@ +// +build ignore + +package pty + +/* +#include +#include +#include +*/ +import "C" + +type ptmget C.struct_ptmget + +var ioctl_PTMGET = C.PTMGET diff --git a/vendor/github.com/kr/pty/util.go b/vendor/github.com/kr/pty/util.go index a4fab9a7ce44..68a8584cfeb8 100644 --- a/vendor/github.com/kr/pty/util.go +++ b/vendor/github.com/kr/pty/util.go @@ -8,26 +8,53 @@ import ( "unsafe" ) +// InheritSize applies the terminal size of master to slave. This should be run +// in a signal handler for syscall.SIGWINCH to automatically resize the slave when +// the master receives a window size change notification. +func InheritSize(master, slave *os.File) error { + size, err := GetsizeFull(master) + if err != nil { + return err + } + err = Setsize(slave, size) + if err != nil { + return err + } + return nil +} + +// Setsize resizes t to s. +func Setsize(t *os.File, ws *Winsize) error { + return windowRectCall(ws, t.Fd(), syscall.TIOCSWINSZ) +} + +// GetsizeFull returns the full terminal size description. +func GetsizeFull(t *os.File) (size *Winsize, err error) { + var ws Winsize + err = windowRectCall(&ws, t.Fd(), syscall.TIOCGWINSZ) + return &ws, err +} + // Getsize returns the number of rows (lines) and cols (positions // in each line) in terminal t. func Getsize(t *os.File) (rows, cols int, err error) { - var ws winsize - err = windowrect(&ws, t.Fd()) - return int(ws.ws_row), int(ws.ws_col), err + ws, err := GetsizeFull(t) + return int(ws.Rows), int(ws.Cols), err } -type winsize struct { - ws_row uint16 - ws_col uint16 - ws_xpixel uint16 - ws_ypixel uint16 +// Winsize describes the terminal size. +type Winsize struct { + Rows uint16 // ws_row: Number of rows (in cells) + Cols uint16 // ws_col: Number of columns (in cells) + X uint16 // ws_xpixel: Width in pixels + Y uint16 // ws_ypixel: Height in pixels } -func windowrect(ws *winsize, fd uintptr) error { +func windowRectCall(ws *Winsize, fd, a2 uintptr) error { _, _, errno := syscall.Syscall( syscall.SYS_IOCTL, fd, - syscall.TIOCGWINSZ, + a2, uintptr(unsafe.Pointer(ws)), ) if errno != 0 { diff --git a/vendor/github.com/kr/pty/ztypes_openbsd_amd64.go b/vendor/github.com/kr/pty/ztypes_openbsd_amd64.go new file mode 100644 index 000000000000..e67051688f00 --- /dev/null +++ b/vendor/github.com/kr/pty/ztypes_openbsd_amd64.go @@ -0,0 +1,13 @@ +// Created by cgo -godefs - DO NOT EDIT +// cgo -godefs types_openbsd.go + +package pty + +type ptmget struct { + Cfd int32 + Sfd int32 + Cn [16]int8 + Sn [16]int8 +} + +var ioctl_PTMGET = 0x40287401 diff --git a/vendor/github.com/kr/text/License b/vendor/github.com/kr/text/License new file mode 100644 index 000000000000..480a32805999 --- /dev/null +++ b/vendor/github.com/kr/text/License @@ -0,0 +1,19 @@ +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/kr/text/doc.go b/vendor/github.com/kr/text/doc.go new file mode 100644 index 000000000000..cf4c198f9558 --- /dev/null +++ b/vendor/github.com/kr/text/doc.go @@ -0,0 +1,3 @@ +// Package text provides rudimentary functions for manipulating text in +// paragraphs. +package text diff --git a/vendor/github.com/kr/text/indent.go b/vendor/github.com/kr/text/indent.go new file mode 100644 index 000000000000..4ebac45c0925 --- /dev/null +++ b/vendor/github.com/kr/text/indent.go @@ -0,0 +1,74 @@ +package text + +import ( + "io" +) + +// Indent inserts prefix at the beginning of each non-empty line of s. The +// end-of-line marker is NL. +func Indent(s, prefix string) string { + return string(IndentBytes([]byte(s), []byte(prefix))) +} + +// IndentBytes inserts prefix at the beginning of each non-empty line of b. +// The end-of-line marker is NL. +func IndentBytes(b, prefix []byte) []byte { + var res []byte + bol := true + for _, c := range b { + if bol && c != '\n' { + res = append(res, prefix...) + } + res = append(res, c) + bol = c == '\n' + } + return res +} + +// Writer indents each line of its input. +type indentWriter struct { + w io.Writer + bol bool + pre [][]byte + sel int + off int +} + +// NewIndentWriter makes a new write filter that indents the input +// lines. Each line is prefixed in order with the corresponding +// element of pre. If there are more lines than elements, the last +// element of pre is repeated for each subsequent line. +func NewIndentWriter(w io.Writer, pre ...[]byte) io.Writer { + return &indentWriter{ + w: w, + pre: pre, + bol: true, + } +} + +// The only errors returned are from the underlying indentWriter. +func (w *indentWriter) Write(p []byte) (n int, err error) { + for _, c := range p { + if w.bol { + var i int + i, err = w.w.Write(w.pre[w.sel][w.off:]) + w.off += i + if err != nil { + return n, err + } + } + _, err = w.w.Write([]byte{c}) + if err != nil { + return n, err + } + n++ + w.bol = c == '\n' + if w.bol { + w.off = 0 + if w.sel < len(w.pre)-1 { + w.sel++ + } + } + } + return n, nil +} diff --git a/vendor/github.com/kr/text/wrap.go b/vendor/github.com/kr/text/wrap.go new file mode 100644 index 000000000000..b09bb03736dc --- /dev/null +++ b/vendor/github.com/kr/text/wrap.go @@ -0,0 +1,86 @@ +package text + +import ( + "bytes" + "math" +) + +var ( + nl = []byte{'\n'} + sp = []byte{' '} +) + +const defaultPenalty = 1e5 + +// Wrap wraps s into a paragraph of lines of length lim, with minimal +// raggedness. +func Wrap(s string, lim int) string { + return string(WrapBytes([]byte(s), lim)) +} + +// WrapBytes wraps b into a paragraph of lines of length lim, with minimal +// raggedness. +func WrapBytes(b []byte, lim int) []byte { + words := bytes.Split(bytes.Replace(bytes.TrimSpace(b), nl, sp, -1), sp) + var lines [][]byte + for _, line := range WrapWords(words, 1, lim, defaultPenalty) { + lines = append(lines, bytes.Join(line, sp)) + } + return bytes.Join(lines, nl) +} + +// WrapWords is the low-level line-breaking algorithm, useful if you need more +// control over the details of the text wrapping process. For most uses, either +// Wrap or WrapBytes will be sufficient and more convenient. +// +// WrapWords splits a list of words into lines with minimal "raggedness", +// treating each byte as one unit, accounting for spc units between adjacent +// words on each line, and attempting to limit lines to lim units. Raggedness +// is the total error over all lines, where error is the square of the +// difference of the length of the line and lim. Too-long lines (which only +// happen when a single word is longer than lim units) have pen penalty units +// added to the error. +func WrapWords(words [][]byte, spc, lim, pen int) [][][]byte { + n := len(words) + + length := make([][]int, n) + for i := 0; i < n; i++ { + length[i] = make([]int, n) + length[i][i] = len(words[i]) + for j := i + 1; j < n; j++ { + length[i][j] = length[i][j-1] + spc + len(words[j]) + } + } + + nbrk := make([]int, n) + cost := make([]int, n) + for i := range cost { + cost[i] = math.MaxInt32 + } + for i := n - 1; i >= 0; i-- { + if length[i][n-1] <= lim || i == n-1 { + cost[i] = 0 + nbrk[i] = n + } else { + for j := i + 1; j < n; j++ { + d := lim - length[i][j-1] + c := d*d + cost[j] + if length[i][j-1] > lim { + c += pen // too-long lines get a worse penalty + } + if c < cost[i] { + cost[i] = c + nbrk[i] = j + } + } + } + } + + var lines [][][]byte + i := 0 + for i < n { + lines = append(lines, words[i:nbrk[i]]) + i = nbrk[i] + } + return lines +} diff --git a/vendor/github.com/motomux/pretty/License b/vendor/github.com/motomux/pretty/License new file mode 100644 index 000000000000..05c783ccf68a --- /dev/null +++ b/vendor/github.com/motomux/pretty/License @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/motomux/pretty/diff.go b/vendor/github.com/motomux/pretty/diff.go new file mode 100644 index 000000000000..125e7eb75c1b --- /dev/null +++ b/vendor/github.com/motomux/pretty/diff.go @@ -0,0 +1,273 @@ +package pretty + +import ( + "fmt" + "io" + "reflect" +) + +type sbuf []string + +func (p *sbuf) Printf(format string, a ...interface{}) { + s := fmt.Sprintf(format, a...) + *p = append(*p, s) +} + +// Diff returns a slice where each element describes +// a difference between a and b. +func Diff(a, b interface{}) (desc []string) { + Pdiff((*sbuf)(&desc), a, b) + return desc +} + +// wprintfer calls Fprintf on w for each Printf call +// with a trailing newline. +type wprintfer struct{ w io.Writer } + +func (p *wprintfer) Printf(format string, a ...interface{}) { + fmt.Fprintf(p.w, format+"\n", a...) +} + +// Fdiff writes to w a description of the differences between a and b. +func Fdiff(w io.Writer, a, b interface{}) { + Pdiff(&wprintfer{w}, a, b) +} + +type Printfer interface { + Printf(format string, a ...interface{}) +} + +// Pdiff prints to p a description of the differences between a and b. +// It calls Printf once for each difference, with no trailing newline. +// The standard library log.Logger is a Printfer. +func Pdiff(p Printfer, a, b interface{}) { + diffPrinter{w: p}.diff(reflect.ValueOf(a), reflect.ValueOf(b)) +} + +type Logfer interface { + Logf(format string, a ...interface{}) +} + +// logprintfer calls Fprintf on w for each Printf call +// with a trailing newline. +type logprintfer struct{ l Logfer } + +func (p *logprintfer) Printf(format string, a ...interface{}) { + p.l.Logf(format, a...) +} + +// Ldiff prints to l a description of the differences between a and b. +// It calls Logf once for each difference, with no trailing newline. +// The standard library testing.T and testing.B are Logfers. +func Ldiff(l Logfer, a, b interface{}) { + Pdiff(&logprintfer{l}, a, b) +} + +type diffPrinter struct { + w Printfer + l string // label +} + +func (w diffPrinter) printf(f string, a ...interface{}) { + var l string + if w.l != "" { + l = w.l + ": " + } + w.w.Printf(l+f, a...) +} + +func (w diffPrinter) diff(av, bv reflect.Value) { + if !av.IsValid() && bv.IsValid() { + w.printf("nil != %# v", formatter{v: bv, quote: true}) + return + } + if av.IsValid() && !bv.IsValid() { + w.printf("%# v != nil", formatter{v: av, quote: true}) + return + } + if !av.IsValid() && !bv.IsValid() { + return + } + + at := av.Type() + bt := bv.Type() + if at != bt { + w.printf("%v != %v", at, bt) + return + } + + switch kind := at.Kind(); kind { + case reflect.Bool: + if a, b := av.Bool(), bv.Bool(); a != b { + w.printf("%v != %v", a, b) + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if a, b := av.Int(), bv.Int(); a != b { + w.printf("%d != %d", a, b) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + if a, b := av.Uint(), bv.Uint(); a != b { + w.printf("%d != %d", a, b) + } + case reflect.Float32, reflect.Float64: + if a, b := av.Float(), bv.Float(); a != b { + w.printf("%v != %v", a, b) + } + case reflect.Complex64, reflect.Complex128: + if a, b := av.Complex(), bv.Complex(); a != b { + w.printf("%v != %v", a, b) + } + case reflect.Array: + n := av.Len() + for i := 0; i < n; i++ { + w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i)) + } + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + if a, b := av.Pointer(), bv.Pointer(); a != b { + w.printf("%#x != %#x", a, b) + } + case reflect.Interface: + w.diff(av.Elem(), bv.Elem()) + case reflect.Map: + ak, both, bk := keyDiff(av.MapKeys(), bv.MapKeys()) + for _, k := range ak { + w := w.relabel(fmt.Sprintf("[%#v]", k)) + w.printf("%q != (missing)", av.MapIndex(k)) + } + for _, k := range both { + w := w.relabel(fmt.Sprintf("[%#v]", k)) + w.diff(av.MapIndex(k), bv.MapIndex(k)) + } + for _, k := range bk { + w := w.relabel(fmt.Sprintf("[%#v]", k)) + w.printf("(missing) != %q", bv.MapIndex(k)) + } + if av.IsNil() != bv.IsNil() { + w.printf("%#v != %#v", av, bv) + break + } + case reflect.Ptr: + switch { + case av.IsNil() && !bv.IsNil(): + w.printf("nil != %# v", formatter{v: bv, quote: true}) + case !av.IsNil() && bv.IsNil(): + w.printf("%# v != nil", formatter{v: av, quote: true}) + case !av.IsNil() && !bv.IsNil(): + w.diff(av.Elem(), bv.Elem()) + } + case reflect.Slice: + lenA := av.Len() + lenB := bv.Len() + if lenA != lenB { + w.printf("%s[%d] != %s[%d]", av.Type(), lenA, bv.Type(), lenB) + break + } + for i := 0; i < lenA; i++ { + w.relabel(fmt.Sprintf("[%d]", i)).diff(av.Index(i), bv.Index(i)) + } + if av.IsNil() != bv.IsNil() { + w.printf("%#v != %#v", av, bv) + break + } + case reflect.String: + if a, b := av.String(), bv.String(); a != b { + w.printf("%q != %q", a, b) + } + case reflect.Struct: + for i := 0; i < av.NumField(); i++ { + w.relabel(at.Field(i).Name).diff(av.Field(i), bv.Field(i)) + } + default: + panic("unknown reflect Kind: " + kind.String()) + } +} + +func (d diffPrinter) relabel(name string) (d1 diffPrinter) { + d1 = d + if d.l != "" && name[0] != '[' { + d1.l += "." + } + d1.l += name + return d1 +} + +// keyEqual compares a and b for equality. +// Both a and b must be valid map keys. +func keyEqual(av, bv reflect.Value) bool { + if !av.IsValid() && !bv.IsValid() { + return true + } + if !av.IsValid() || !bv.IsValid() || av.Type() != bv.Type() { + return false + } + switch kind := av.Kind(); kind { + case reflect.Bool: + a, b := av.Bool(), bv.Bool() + return a == b + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + a, b := av.Int(), bv.Int() + return a == b + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + a, b := av.Uint(), bv.Uint() + return a == b + case reflect.Float32, reflect.Float64: + a, b := av.Float(), bv.Float() + return a == b + case reflect.Complex64, reflect.Complex128: + a, b := av.Complex(), bv.Complex() + return a == b + case reflect.Array: + for i := 0; i < av.Len(); i++ { + if !keyEqual(av.Index(i), bv.Index(i)) { + return false + } + } + return true + case reflect.Chan, reflect.UnsafePointer, reflect.Ptr: + a, b := av.Pointer(), bv.Pointer() + return a == b + case reflect.Interface: + return keyEqual(av.Elem(), bv.Elem()) + case reflect.String: + a, b := av.String(), bv.String() + return a == b + case reflect.Struct: + for i := 0; i < av.NumField(); i++ { + if !keyEqual(av.Field(i), bv.Field(i)) { + return false + } + } + return true + default: + panic("invalid map key type " + av.Type().String()) + } +} + +func keyDiff(a, b []reflect.Value) (ak, both, bk []reflect.Value) { + for _, av := range a { + inBoth := false + for _, bv := range b { + if keyEqual(av, bv) { + inBoth = true + both = append(both, av) + break + } + } + if !inBoth { + ak = append(ak, av) + } + } + for _, bv := range b { + inBoth := false + for _, av := range a { + if keyEqual(av, bv) { + inBoth = true + break + } + } + if !inBoth { + bk = append(bk, bv) + } + } + return +} diff --git a/vendor/github.com/motomux/pretty/formatter.go b/vendor/github.com/motomux/pretty/formatter.go new file mode 100644 index 000000000000..a317d7b8ee88 --- /dev/null +++ b/vendor/github.com/motomux/pretty/formatter.go @@ -0,0 +1,328 @@ +package pretty + +import ( + "fmt" + "io" + "reflect" + "strconv" + "text/tabwriter" + + "github.com/kr/text" +) + +type formatter struct { + v reflect.Value + force bool + quote bool +} + +// Formatter makes a wrapper, f, that will format x as go source with line +// breaks and tabs. Object f responds to the "%v" formatting verb when both the +// "#" and " " (space) flags are set, for example: +// +// fmt.Sprintf("%# v", Formatter(x)) +// +// If one of these two flags is not set, or any other verb is used, f will +// format x according to the usual rules of package fmt. +// In particular, if x satisfies fmt.Formatter, then x.Format will be called. +func Formatter(x interface{}) (f fmt.Formatter) { + return formatter{v: reflect.ValueOf(x), quote: true} +} + +func (fo formatter) String() string { + return fmt.Sprint(fo.v.Interface()) // unwrap it +} + +func (fo formatter) passThrough(f fmt.State, c rune) { + s := "%" + for i := 0; i < 128; i++ { + if f.Flag(i) { + s += string(i) + } + } + if w, ok := f.Width(); ok { + s += fmt.Sprintf("%d", w) + } + if p, ok := f.Precision(); ok { + s += fmt.Sprintf(".%d", p) + } + s += string(c) + fmt.Fprintf(f, s, fo.v.Interface()) +} + +func (fo formatter) Format(f fmt.State, c rune) { + if fo.force || c == 'v' && f.Flag('#') && f.Flag(' ') { + w := tabwriter.NewWriter(f, 4, 4, 1, ' ', 0) + p := &printer{tw: w, Writer: w, visited: make(map[visit]int)} + p.printValue(fo.v, true, fo.quote) + w.Flush() + return + } + fo.passThrough(f, c) +} + +type printer struct { + io.Writer + tw *tabwriter.Writer + visited map[visit]int + depth int +} + +func (p *printer) indent() *printer { + q := *p + q.tw = tabwriter.NewWriter(p.Writer, 4, 4, 1, ' ', 0) + q.Writer = text.NewIndentWriter(q.tw, []byte{'\t'}) + return &q +} + +func (p *printer) printInline(v reflect.Value, x interface{}, showType bool) { + if showType { + io.WriteString(p, v.Type().String()) + fmt.Fprintf(p, "(%#v)", x) + } else { + fmt.Fprintf(p, "%#v", x) + } +} + +// printValue must keep track of already-printed pointer values to avoid +// infinite recursion. +type visit struct { + v uintptr + typ reflect.Type +} + +func (p *printer) printValue(v reflect.Value, showType, quote bool) { + if p.depth > 10 { + io.WriteString(p, "!%v(DEPTH EXCEEDED)") + return + } + + switch v.Kind() { + case reflect.Bool: + p.printInline(v, v.Bool(), showType) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + p.printInline(v, v.Int(), showType) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + p.printInline(v, v.Uint(), showType) + case reflect.Float32, reflect.Float64: + p.printInline(v, v.Float(), showType) + case reflect.Complex64, reflect.Complex128: + fmt.Fprintf(p, "%#v", v.Complex()) + case reflect.String: + p.fmtString(v.String(), quote) + case reflect.Map: + t := v.Type() + if showType { + io.WriteString(p, t.String()) + } + writeByte(p, '{') + if nonzero(v) { + expand := !canInline(v.Type()) + pp := p + if expand { + writeByte(p, '\n') + pp = p.indent() + } + keys := v.MapKeys() + for i := 0; i < v.Len(); i++ { + showTypeInStruct := true + k := keys[i] + mv := v.MapIndex(k) + pp.printValue(k, false, true) + writeByte(pp, ':') + if expand { + writeByte(pp, '\t') + } + showTypeInStruct = t.Elem().Kind() == reflect.Interface + pp.printValue(mv, showTypeInStruct, true) + if expand { + io.WriteString(pp, ",\n") + } else if i < v.Len()-1 { + io.WriteString(pp, ", ") + } + } + if expand { + pp.tw.Flush() + } + } + writeByte(p, '}') + case reflect.Struct: + t := v.Type() + if v.CanAddr() { + addr := v.UnsafeAddr() + vis := visit{addr, t} + if vd, ok := p.visited[vis]; ok && vd < p.depth { + p.fmtString(t.String()+"{(CYCLIC REFERENCE)}", false) + break // don't print v again + } + p.visited[vis] = p.depth + } + + if showType { + io.WriteString(p, t.String()) + } + writeByte(p, '{') + if nonzero(v) { + expand := !canInline(v.Type()) + pp := p + if expand { + writeByte(p, '\n') + pp = p.indent() + } + for i := 0; i < v.NumField(); i++ { + showTypeInStruct := true + if f := t.Field(i); f.Name != "" { + io.WriteString(pp, f.Name) + writeByte(pp, ':') + if expand { + writeByte(pp, '\t') + } + showTypeInStruct = labelType(f.Type) + } + pp.printValue(getField(v, i), showTypeInStruct, true) + if expand { + io.WriteString(pp, ",\n") + } else if i < v.NumField()-1 { + io.WriteString(pp, ", ") + } + } + if expand { + pp.tw.Flush() + } + } + writeByte(p, '}') + case reflect.Interface: + switch e := v.Elem(); { + case e.Kind() == reflect.Invalid: + io.WriteString(p, "nil") + case e.IsValid(): + pp := *p + pp.depth++ + pp.printValue(e, showType, true) + default: + io.WriteString(p, v.Type().String()) + io.WriteString(p, "(nil)") + } + case reflect.Array, reflect.Slice: + t := v.Type() + if showType { + io.WriteString(p, t.String()) + } + if v.Kind() == reflect.Slice && v.IsNil() && showType { + io.WriteString(p, "(nil)") + break + } + if v.Kind() == reflect.Slice && v.IsNil() { + io.WriteString(p, "nil") + break + } + writeByte(p, '{') + expand := !canInline(v.Type()) + pp := p + if expand { + writeByte(p, '\n') + pp = p.indent() + } + for i := 0; i < v.Len(); i++ { + showTypeInSlice := t.Elem().Kind() == reflect.Interface + pp.printValue(v.Index(i), showTypeInSlice, true) + if expand { + io.WriteString(pp, ",\n") + } else if i < v.Len()-1 { + io.WriteString(pp, ", ") + } + } + if expand { + pp.tw.Flush() + } + writeByte(p, '}') + case reflect.Ptr: + e := v.Elem() + if !e.IsValid() { + writeByte(p, '(') + io.WriteString(p, v.Type().String()) + io.WriteString(p, ")(nil)") + } else { + pp := *p + pp.depth++ + writeByte(pp, '&') + pp.printValue(e, true, true) + } + case reflect.Chan: + x := v.Pointer() + if showType { + writeByte(p, '(') + io.WriteString(p, v.Type().String()) + fmt.Fprintf(p, ")(%#v)", x) + } else { + fmt.Fprintf(p, "%#v", x) + } + case reflect.Func: + io.WriteString(p, v.Type().String()) + io.WriteString(p, " {...}") + case reflect.UnsafePointer: + p.printInline(v, v.Pointer(), showType) + case reflect.Invalid: + io.WriteString(p, "nil") + } +} + +func canInline(t reflect.Type) bool { + switch t.Kind() { + case reflect.Map: + return !canExpand(t.Elem()) + case reflect.Struct: + for i := 0; i < t.NumField(); i++ { + if canExpand(t.Field(i).Type) { + return false + } + } + return true + case reflect.Interface: + return false + case reflect.Array, reflect.Slice: + return !canExpand(t.Elem()) + case reflect.Ptr: + return false + case reflect.Chan, reflect.Func, reflect.UnsafePointer: + return false + } + return true +} + +func canExpand(t reflect.Type) bool { + switch t.Kind() { + case reflect.Map, reflect.Struct, + reflect.Interface, reflect.Array, reflect.Slice, + reflect.Ptr: + return true + } + return false +} + +func labelType(t reflect.Type) bool { + switch t.Kind() { + case reflect.Interface, reflect.Struct: + return true + } + return false +} + +func (p *printer) fmtString(s string, quote bool) { + if quote { + s = strconv.Quote(s) + } + io.WriteString(p, s) +} + +func writeByte(w io.Writer, b byte) { + w.Write([]byte{b}) +} + +func getField(v reflect.Value, i int) reflect.Value { + val := v.Field(i) + if val.Kind() == reflect.Interface && !val.IsNil() { + val = val.Elem() + } + return val +} diff --git a/vendor/github.com/motomux/pretty/pretty.go b/vendor/github.com/motomux/pretty/pretty.go new file mode 100644 index 000000000000..49423ec7f540 --- /dev/null +++ b/vendor/github.com/motomux/pretty/pretty.go @@ -0,0 +1,108 @@ +// Package pretty provides pretty-printing for Go values. This is +// useful during debugging, to avoid wrapping long output lines in +// the terminal. +// +// It provides a function, Formatter, that can be used with any +// function that accepts a format string. It also provides +// convenience wrappers for functions in packages fmt and log. +package pretty + +import ( + "fmt" + "io" + "log" + "reflect" +) + +// Errorf is a convenience wrapper for fmt.Errorf. +// +// Calling Errorf(f, x, y) is equivalent to +// fmt.Errorf(f, Formatter(x), Formatter(y)). +func Errorf(format string, a ...interface{}) error { + return fmt.Errorf(format, wrap(a, false)...) +} + +// Fprintf is a convenience wrapper for fmt.Fprintf. +// +// Calling Fprintf(w, f, x, y) is equivalent to +// fmt.Fprintf(w, f, Formatter(x), Formatter(y)). +func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error error) { + return fmt.Fprintf(w, format, wrap(a, false)...) +} + +// Log is a convenience wrapper for log.Printf. +// +// Calling Log(x, y) is equivalent to +// log.Print(Formatter(x), Formatter(y)), but each operand is +// formatted with "%# v". +func Log(a ...interface{}) { + log.Print(wrap(a, true)...) +} + +// Logf is a convenience wrapper for log.Printf. +// +// Calling Logf(f, x, y) is equivalent to +// log.Printf(f, Formatter(x), Formatter(y)). +func Logf(format string, a ...interface{}) { + log.Printf(format, wrap(a, false)...) +} + +// Logln is a convenience wrapper for log.Printf. +// +// Calling Logln(x, y) is equivalent to +// log.Println(Formatter(x), Formatter(y)), but each operand is +// formatted with "%# v". +func Logln(a ...interface{}) { + log.Println(wrap(a, true)...) +} + +// Print pretty-prints its operands and writes to standard output. +// +// Calling Print(x, y) is equivalent to +// fmt.Print(Formatter(x), Formatter(y)), but each operand is +// formatted with "%# v". +func Print(a ...interface{}) (n int, errno error) { + return fmt.Print(wrap(a, true)...) +} + +// Printf is a convenience wrapper for fmt.Printf. +// +// Calling Printf(f, x, y) is equivalent to +// fmt.Printf(f, Formatter(x), Formatter(y)). +func Printf(format string, a ...interface{}) (n int, errno error) { + return fmt.Printf(format, wrap(a, false)...) +} + +// Println pretty-prints its operands and writes to standard output. +// +// Calling Print(x, y) is equivalent to +// fmt.Println(Formatter(x), Formatter(y)), but each operand is +// formatted with "%# v". +func Println(a ...interface{}) (n int, errno error) { + return fmt.Println(wrap(a, true)...) +} + +// Sprint is a convenience wrapper for fmt.Sprintf. +// +// Calling Sprint(x, y) is equivalent to +// fmt.Sprint(Formatter(x), Formatter(y)), but each operand is +// formatted with "%# v". +func Sprint(a ...interface{}) string { + return fmt.Sprint(wrap(a, true)...) +} + +// Sprintf is a convenience wrapper for fmt.Sprintf. +// +// Calling Sprintf(f, x, y) is equivalent to +// fmt.Sprintf(f, Formatter(x), Formatter(y)). +func Sprintf(format string, a ...interface{}) string { + return fmt.Sprintf(format, wrap(a, false)...) +} + +func wrap(a []interface{}, force bool) []interface{} { + w := make([]interface{}, len(a)) + for i, x := range a { + w[i] = formatter{v: reflect.ValueOf(x), force: force} + } + return w +} diff --git a/vendor/github.com/motomux/pretty/zero.go b/vendor/github.com/motomux/pretty/zero.go new file mode 100644 index 000000000000..abb5b6fc14cc --- /dev/null +++ b/vendor/github.com/motomux/pretty/zero.go @@ -0,0 +1,41 @@ +package pretty + +import ( + "reflect" +) + +func nonzero(v reflect.Value) bool { + switch v.Kind() { + case reflect.Bool: + return v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() != 0 + case reflect.Float32, reflect.Float64: + return v.Float() != 0 + case reflect.Complex64, reflect.Complex128: + return v.Complex() != complex(0, 0) + case reflect.String: + return v.String() != "" + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if nonzero(getField(v, i)) { + return true + } + } + return false + case reflect.Array: + for i := 0; i < v.Len(); i++ { + if nonzero(v.Index(i)) { + return true + } + } + return false + case reflect.Map, reflect.Interface, reflect.Slice, reflect.Ptr, reflect.Chan, reflect.Func: + return !v.IsNil() + case reflect.UnsafePointer: + return v.Pointer() != 0 + } + return true +}