diff --git a/.circleci/config.yml b/.circleci/config.yml index 6c29056..84abacd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,11 +11,11 @@ jobs: command: | echo 'export GO111MODULE=on' >> $BASH_ENV - run: - name: "Run Tests" - command: go test -v ./... + name: "Run All Tests with Race Detection" + command: make testrace - run: name: "Install Gox" command: go get github.com/mitchellh/gox - run: name: "Run Build" - command: ./scripts/build.sh + command: make dev diff --git a/Makefile b/Makefile index 76cb650..533c9c9 100644 --- a/Makefile +++ b/Makefile @@ -19,12 +19,12 @@ dev: fmtcheck generate @CGO_ENABLED=0 BUILD_TAGS='$(BUILD_TAGS)' VAULT_DEV_BUILD=1 sh -c "'$(CURDIR)/scripts/build.sh'" # testshort runs the quick unit tests and vets the code -testshort: fmtcheck generate +test: fmtcheck generate CGO_ENABLED=0 VAULT_TOKEN= VAULT_ACC= go test -v -short -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -count=1 -timeout=20m -parallel=4 # test runs the unit tests and vets the code -test: fmtcheck generate - CGO_ENABLED=0 VAULT_TOKEN= VAULT_ACC= go test -v -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -count=1 -timeout=20m -parallel=4 +testrace: fmtcheck generate + CGO_ENABLED=1 VAULT_TOKEN= VAULT_ACC= go test -race -v -tags='$(BUILD_TAGS)' $(TEST) $(TESTARGS) -count=1 -timeout=20m -parallel=4 testcompile: fmtcheck generate @for pkg in $(TEST) ; do \ diff --git a/go.mod b/go.mod index ba15f3a..ef036ec 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/hashicorp/vault-plugin-secrets-ad go 1.12 require ( + github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da github.com/go-errors/errors v1.0.1 github.com/go-ldap/ldap v3.0.2+incompatible github.com/hashicorp/go-hclog v0.8.0 diff --git a/go.sum b/go.sum index dfcf046..44ed4cc 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= diff --git a/plugin/backend.go b/plugin/backend.go index 979b6aa..6174e15 100644 --- a/plugin/backend.go +++ b/plugin/backend.go @@ -8,13 +8,16 @@ import ( "github.com/hashicorp/vault-plugin-secrets-ad/plugin/client" "github.com/hashicorp/vault-plugin-secrets-ad/plugin/util" "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/locksutil" "github.com/hashicorp/vault/sdk/logical" "github.com/patrickmn/go-cache" ) func Factory(ctx context.Context, conf *logical.BackendConfig) (logical.Backend, error) { backend := newBackend(util.NewSecretsClient(conf.Logger)) - backend.Setup(ctx, conf) + if err := backend.Setup(ctx, conf); err != nil { + return nil, err + } return backend, nil } @@ -24,6 +27,10 @@ func newBackend(client secretsClient) *backend { roleCache: cache.New(roleCacheExpiration, roleCacheCleanup), credCache: cache.New(credCacheExpiration, credCacheCleanup), rotateRootLock: new(int32), + checkOutHandler: &checkOutHandler{ + client: client, + }, + checkOutLocks: locksutil.CreateLocks(), } adBackend.Backend = &framework.Backend{ Help: backendHelp, @@ -33,6 +40,14 @@ func newBackend(client secretsClient) *backend { adBackend.pathListRoles(), adBackend.pathCreds(), adBackend.pathRotateCredentials(), + + // The following paths are for AD credential checkout. + adBackend.pathSetCheckIn(), + adBackend.pathSetManageCheckIn(), + adBackend.pathSetCheckOut(), + adBackend.pathSetStatus(), + adBackend.pathSets(), + adBackend.pathListSets(), }, PathsSpecial: &logical.Paths{ SealWrapStorage: []string{ @@ -42,12 +57,15 @@ func newBackend(client secretsClient) *backend { }, Invalidate: adBackend.Invalidate, BackendType: logical.TypeLogical, + Secrets: []*framework.Secret{ + adBackend.secretAccessKeys(), + }, } return adBackend } type backend struct { - logical.Backend + *framework.Backend client secretsClient @@ -55,6 +73,11 @@ type backend struct { credCache *cache.Cache credLock sync.Mutex rotateRootLock *int32 + + checkOutHandler *checkOutHandler + // checkOutLocks are used for avoiding races + // when working with sets through the check-out system. + checkOutLocks []*locksutil.LockEntry } func (b *backend) Invalidate(ctx context.Context, key string) { diff --git a/plugin/backend_checkouts_test.go b/plugin/backend_checkouts_test.go new file mode 100644 index 0000000..bec574d --- /dev/null +++ b/plugin/backend_checkouts_test.go @@ -0,0 +1,527 @@ +package plugin + +import ( + "testing" + "time" + + "github.com/hashicorp/vault/sdk/logical" +) + +// The AD library of service accounts that can be checked out +// is a discrete set of features. This test suite provides +// end-to-end tests of these interrelated endpoints. +func TestCheckOuts(t *testing.T) { + // Plant a config. + t.Run("plant config", PlantConfig) + + // Exercise all set endpoints. + t.Run("write set", WriteSet) + t.Run("read set", ReadSet) + t.Run("read set status", ReadSetStatus) + t.Run("write set toggle off", WriteSetToggleOff) + t.Run("read set toggle off", ReadSetToggleOff) + t.Run("write conflicting set", WriteSetWithConflictingServiceAccounts) + t.Run("list sets", ListSets) + t.Run("delete set", DeleteSet) + + // Do some common updates on sets and ensure they work. + t.Run("write set", WriteSet) + t.Run("add service account", AddAnotherServiceAccount) + t.Run("remove service account", RemoveServiceAccount) + + t.Run("check initial status", CheckInitialStatus) + t.Run("check out account", PerformCheckOut) + t.Run("check updated status", CheckUpdatedStatus) + t.Run("normal check in", NormalCheckIn) + t.Run("return to initial status", CheckInitialStatus) + t.Run("check out again", PerformCheckOut) + t.Run("check updated status", CheckUpdatedStatus) + t.Run("force check in", ForceCheckIn) + t.Run("check all are available", CheckInitialStatus) +} + +// TestCheckOutRaces executes a whole bunch of calls at once and only looks for +// races. Responses are ignored because they'll vary depending on execution order. +func TestCheckOutRaces(t *testing.T) { + if testing.Short() { + t.Skip("skipping check for races in the checkout system due to short flag") + } + + // Get 100 goroutines ready to go. + numParallel := 100 + start := make(chan bool, 1) + end := make(chan bool, numParallel) + for i := 0; i < numParallel; i++ { + go func() { + <-start + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.CreateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + "ttl": "10h", + "max_ttl": "11h", + "disable_check_in_enforcement": true, + }, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com", "tester3@example.com"}, + }, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + }, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + "ttl": "10h", + "disable_check_in_enforcement": false, + }, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.CreateOperation, + Path: libraryPrefix + "test-set2", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": "tester1@example.com", + }, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ListOperation, + Path: libraryPrefix, + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.DeleteOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/check-out", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/check-in", + Storage: testStorage, + }) + testBackend.HandleRequest(ctx, &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "manage/test-set/check-in", + Storage: testStorage, + }) + end <- true + }() + } + + // Start them all at once. + close(start) + + // Wait for them all to finish. + timer := time.NewTimer(15 * time.Second) + for i := 0; i < numParallel; i++ { + select { + case <-timer.C: + t.Fatal("test took more than 15 seconds, may be deadlocked") + case <-end: + continue + } + } +} + +func WriteSet(t *testing.T) { + req := &logical.Request{ + Operation: logical.CreateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + "ttl": "10h", + "max_ttl": "11h", + "disable_check_in_enforcement": true, + }, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp != nil { + t.Fatalf("expected an empty response, got: %v", resp) + } +} + +func AddAnotherServiceAccount(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com", "tester3@example.com"}, + }, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp != nil { + t.Fatalf("expected an empty response, got: %v", resp) + } +} + +func RemoveServiceAccount(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + }, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp != nil { + t.Fatalf("expected an empty response, got: %v", resp) + } +} + +func ReadSet(t *testing.T) { + req := &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + serviceAccountNames := resp.Data["service_account_names"].([]string) + if len(serviceAccountNames) != 2 { + t.Fatal("expected 2") + } + disableCheckInEnforcement := resp.Data["disable_check_in_enforcement"].(bool) + if !disableCheckInEnforcement { + t.Fatal("check-in enforcement should be disabled") + } + ttl := resp.Data["ttl"].(int64) + if ttl != 10*60*60 { // 10 hours + t.Fatal(ttl) + } + maxTTL := resp.Data["max_ttl"].(int64) + if maxTTL != 11*60*60 { // 11 hours + t.Fatal(maxTTL) + } +} + +func WriteSetToggleOff(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": []string{"tester1@example.com", "tester2@example.com"}, + "ttl": "10h", + "disable_check_in_enforcement": false, + }, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp != nil { + t.Fatalf("expected an empty response, got: %v", resp) + } +} + +func ReadSetToggleOff(t *testing.T) { + req := &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + serviceAccountNames := resp.Data["service_account_names"].([]string) + if len(serviceAccountNames) != 2 { + t.Fatal("expected 2") + } + disableCheckInEnforcement := resp.Data["disable_check_in_enforcement"].(bool) + if disableCheckInEnforcement { + t.Fatal("check-in enforcement should be enabled") + } +} + +func ReadSetStatus(t *testing.T) { + req := &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + if len(resp.Data) != 2 { + t.Fatal("length should be 2 because there are two service accounts in this set") + } + if resp.Data["tester1@example.com"] == nil { + t.Fatal("expected non-nil map") + } + testerStatus := resp.Data["tester1@example.com"].(map[string]interface{}) + if !testerStatus["available"].(bool) { + t.Fatal("should be available for checkout") + } +} + +func WriteSetWithConflictingServiceAccounts(t *testing.T) { + req := &logical.Request{ + Operation: logical.CreateOperation, + Path: libraryPrefix + "test-set2", + Storage: testStorage, + Data: map[string]interface{}{ + "service_account_names": "tester1@example.com", + }, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil { + t.Fatal(err) + } + if resp == nil || !resp.IsError() { + t.Fatal("expected err response because we're adding a service account managed by another set") + } +} + +func ListSets(t *testing.T) { + req := &logical.Request{ + Operation: logical.ListOperation, + Path: libraryPrefix, + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + if resp.Data["keys"] == nil { + t.Fatal("expected non-nil data") + } + listedKeys := resp.Data["keys"].([]string) + if len(listedKeys) != 1 { + t.Fatalf("expected 1 key but received %s", listedKeys) + } + if "test-set" != listedKeys[0] { + t.Fatal("expected test-set to be the only listed item") + } +} + +func DeleteSet(t *testing.T) { + req := &logical.Request{ + Operation: logical.DeleteOperation, + Path: libraryPrefix + "test-set", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp != nil { + t.Fatalf("expected an empty response, got: %v", resp) + } +} + +func CheckInitialStatus(t *testing.T) { + req := &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + if resp.Data["tester1@example.com"] == nil { + t.Fatal("expected map to not be nil") + } + tester1CheckOut := resp.Data["tester1@example.com"].(map[string]interface{}) + available := tester1CheckOut["available"].(bool) + if !available { + t.Fatal("tester1 should be available") + } + + if resp.Data["tester2@example.com"] == nil { + t.Fatal("expected map to not be nil") + } + tester2CheckOut := resp.Data["tester2@example.com"].(map[string]interface{}) + available = tester2CheckOut["available"].(bool) + if !available { + t.Fatal("tester2 should be available") + } +} + +func PerformCheckOut(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set/check-out", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + if resp.Data == nil { + t.Fatal("expected resp data to not be nil") + } + + if resp.Data["service_account_name"] == nil { + t.Fatal("expected string to be populated") + } + if resp.Data["service_account_name"].(string) == "" { + t.Fatal("service account name should be populated") + } + if resp.Data["password"].(string) == "" { + t.Fatal("password should be populated") + } + if !resp.Secret.Renewable { + t.Fatal("lease should be renewable") + } + if resp.Secret.TTL != time.Hour*10 { + t.Fatal("expected 10h TTL") + } + if resp.Secret.MaxTTL != time.Hour*11 { + t.Fatal("expected 11h TTL") + } + if resp.Secret.InternalData["service_account_name"].(string) == "" { + t.Fatal("internal service account name should not be empty") + } +} + +func CheckUpdatedStatus(t *testing.T) { + req := &logical.Request{ + Operation: logical.ReadOperation, + Path: libraryPrefix + "test-set/status", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + if resp.Data == nil { + t.Fatal("expected data to not be nil") + } + + if resp.Data["tester1@example.com"] == nil { + t.Fatal("expected map to not be nil") + } + tester1CheckOut := resp.Data["tester1@example.com"].(map[string]interface{}) + tester1Available := tester1CheckOut["available"].(bool) + + if resp.Data["tester2@example.com"] == nil { + t.Fatal("expected map to not be nil") + } + tester2CheckOut := resp.Data["tester2@example.com"].(map[string]interface{}) + tester2Available := tester2CheckOut["available"].(bool) + + if tester1Available && tester2Available { + t.Fatal("one of the testers should not be available") + } +} + +func NormalCheckIn(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "test-set/check-in", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + checkIns := resp.Data["check_ins"].([]string) + if len(checkIns) != 1 { + t.Fatal("expected 1 check-in") + } +} + +func ForceCheckIn(t *testing.T) { + req := &logical.Request{ + Operation: logical.UpdateOperation, + Path: libraryPrefix + "manage/test-set/check-in", + Storage: testStorage, + } + resp, err := testBackend.HandleRequest(ctx, req) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatal(err) + } + if resp == nil { + t.Fatal("expected a response") + } + checkIns := resp.Data["check_ins"].([]string) + if len(checkIns) != 1 { + t.Fatal("expected 1 check-in") + } +} diff --git a/plugin/backend_test.go b/plugin/backend_test.go index 40662d1..64a35dc 100644 --- a/plugin/backend_test.go +++ b/plugin/backend_test.go @@ -2,6 +2,7 @@ package plugin import ( "context" + "errors" "fmt" "strings" "testing" @@ -23,7 +24,7 @@ var ( MaxLeaseTTLVal: maxLeaseTTLVal, }, } - b := newBackend(&fake{}) + b := newBackend(&fakeSecretsClient{}) b.Setup(context.Background(), conf) return b }() @@ -336,25 +337,43 @@ Beq3QOqp2+dga36IzQybzPQ8QtotrpSJ3q82zztEvyWiJ7E= -----END CERTIFICATE----- ` -type fake struct{} +type fakeSecretsClient struct { + throwErrs bool +} -func (f *fake) Get(conf *client.ADConf, serviceAccountName string) (*client.Entry, error) { +func (f *fakeSecretsClient) Get(conf *client.ADConf, serviceAccountName string) (*client.Entry, error) { entry := &ldap.Entry{} entry.Attributes = append(entry.Attributes, &ldap.EntryAttribute{ Name: client.FieldRegistry.PasswordLastSet.String(), Values: []string{"131680504285591921"}, }) - return client.NewEntry(entry), nil + var err error + if f.throwErrs { + err = errors.New("nope") + } + return client.NewEntry(entry), err } -func (f *fake) GetPasswordLastSet(conf *client.ADConf, serviceAccountName string) (time.Time, error) { - return time.Time{}, nil +func (f *fakeSecretsClient) GetPasswordLastSet(conf *client.ADConf, serviceAccountName string) (time.Time, error) { + var err error + if f.throwErrs { + err = errors.New("nope") + } + return time.Time{}, err } -func (f *fake) UpdatePassword(conf *client.ADConf, serviceAccountName string, newPassword string) error { - return nil +func (f *fakeSecretsClient) UpdatePassword(conf *client.ADConf, serviceAccountName string, newPassword string) error { + var err error + if f.throwErrs { + err = errors.New("nope") + } + return err } -func (f *fake) UpdateRootPassword(conf *client.ADConf, bindDN string, newPassword string) error { - return nil +func (f *fakeSecretsClient) UpdateRootPassword(conf *client.ADConf, bindDN string, newPassword string) error { + var err error + if f.throwErrs { + err = errors.New("nope") + } + return err } diff --git a/plugin/checkout_handler.go b/plugin/checkout_handler.go new file mode 100644 index 0000000..5b4c729 --- /dev/null +++ b/plugin/checkout_handler.go @@ -0,0 +1,192 @@ +package plugin + +import ( + "context" + "errors" + "github.com/hashicorp/vault-plugin-secrets-ad/plugin/util" + "github.com/hashicorp/vault/sdk/logical" +) + +const ( + checkoutStoragePrefix = "checkout/" + passwordStoragePrefix = "password/" +) + +var ( + // errCheckedOut is returned when a check-out request is received + // for a service account that's already checked out. + errCheckedOut = errors.New("checked out") + + // errNotFound is used when a requested item doesn't exist. + errNotFound = errors.New("not found") +) + +// CheckOut provides information for a service account that is currently +// checked out. +type CheckOut struct { + IsAvailable bool `json:"is_available"` + BorrowerEntityID string `json:"borrower_entity_id"` + BorrowerClientToken string `json:"borrower_client_token"` +} + +// checkOutHandler manages checkouts. It's not thread-safe and expects the caller to handle locking because +// locking may span multiple calls. +type checkOutHandler struct { + client secretsClient +} + +// CheckOut attempts to check out a service account. If the account is unavailable, it returns +// errCheckedOut. If the service account isn't managed by this plugin, it returns +// errNotFound. +func (h *checkOutHandler) CheckOut(ctx context.Context, storage logical.Storage, serviceAccountName string, checkOut *CheckOut) error { + if ctx == nil { + return errors.New("ctx must be provided") + } + if storage == nil { + return errors.New("storage must be provided") + } + if serviceAccountName == "" { + return errors.New("service account name must be provided") + } + if checkOut == nil { + return errors.New("check-out must be provided") + } + + // Check if the service account is currently checked out. + currentEntry, err := storage.Get(ctx, checkoutStoragePrefix+serviceAccountName) + if err != nil { + return err + } + if currentEntry == nil { + return errNotFound + } + currentCheckOut := &CheckOut{} + if err := currentEntry.DecodeJSON(currentCheckOut); err != nil { + return err + } + if !currentCheckOut.IsAvailable { + return errCheckedOut + } + + // Since it's not, store the new check-out. + entry, err := logical.StorageEntryJSON(checkoutStoragePrefix+serviceAccountName, checkOut) + if err != nil { + return err + } + return storage.Put(ctx, entry) +} + +// CheckIn attempts to check in a service account. If an error occurs, the account remains checked out +// and can either be retried by the caller, or eventually may be checked in if it has a ttl +// that ends. +func (h *checkOutHandler) CheckIn(ctx context.Context, storage logical.Storage, serviceAccountName string) error { + if ctx == nil { + return errors.New("ctx must be provided") + } + if storage == nil { + return errors.New("storage must be provided") + } + if serviceAccountName == "" { + return errors.New("service account name must be provided") + } + + // On check-ins, a new AD password is generated, updated in AD, and stored. + engineConf, err := readConfig(ctx, storage) + if err != nil { + return err + } + if engineConf == nil { + return errors.New("the config is currently unset") + } + newPassword, err := util.GeneratePassword(engineConf.PasswordConf.Formatter, engineConf.PasswordConf.Length) + if err != nil { + return err + } + if err := h.client.UpdatePassword(engineConf.ADConf, serviceAccountName, newPassword); err != nil { + return err + } + pwdEntry, err := logical.StorageEntryJSON(passwordStoragePrefix+serviceAccountName, newPassword) + if err != nil { + return err + } + if err := storage.Put(ctx, pwdEntry); err != nil { + return err + } + + // That ends the password-handling leg of our journey, now let's deal with the stored check-out itself. + // Store a check-out status indicating it's available. + checkOut := &CheckOut{ + IsAvailable: true, + } + entry, err := logical.StorageEntryJSON(checkoutStoragePrefix+serviceAccountName, checkOut) + if err != nil { + return err + } + return storage.Put(ctx, entry) +} + +// LoadCheckOut returns either: +// - A *CheckOut and nil error if the serviceAccountName is currently managed by this engine. +// - A nil *Checkout and errNotFound if the serviceAccountName is not currently managed by this engine. +func (h *checkOutHandler) LoadCheckOut(ctx context.Context, storage logical.Storage, serviceAccountName string) (*CheckOut, error) { + if ctx == nil { + return nil, errors.New("ctx must be provided") + } + if storage == nil { + return nil, errors.New("storage must be provided") + } + if serviceAccountName == "" { + return nil, errors.New("service account name must be provided") + } + + entry, err := storage.Get(ctx, checkoutStoragePrefix+serviceAccountName) + if err != nil { + return nil, err + } + if entry == nil { + return nil, errNotFound + } + checkOut := &CheckOut{} + if err := entry.DecodeJSON(checkOut); err != nil { + return nil, err + } + return checkOut, nil +} + +// Delete cleans up anything we were tracking from the service account that we will no longer need. +func (h *checkOutHandler) Delete(ctx context.Context, storage logical.Storage, serviceAccountName string) error { + if ctx == nil { + return errors.New("ctx must be provided") + } + if storage == nil { + return errors.New("storage must be provided") + } + if serviceAccountName == "" { + return errors.New("service account name must be provided") + } + + if err := storage.Delete(ctx, passwordStoragePrefix+serviceAccountName); err != nil { + return err + } + return storage.Delete(ctx, checkoutStoragePrefix+serviceAccountName) +} + +// retrievePassword is a utility function for grabbing a service account's password from storage. +// retrievePassword will return: +// - "password", nil if it was successfully able to retrieve the password. +// - errNotFound if there's no password presently. +// - Some other err if it was unable to complete successfully. +func retrievePassword(ctx context.Context, storage logical.Storage, serviceAccountName string) (string, error) { + entry, err := storage.Get(ctx, passwordStoragePrefix+serviceAccountName) + if err != nil { + return "", err + } + if entry == nil { + return "", errNotFound + } + password := "" + if err := entry.DecodeJSON(&password); err != nil { + return "", err + } + return password, nil +} diff --git a/plugin/checkout_handler_test.go b/plugin/checkout_handler_test.go new file mode 100644 index 0000000..33a83ca --- /dev/null +++ b/plugin/checkout_handler_test.go @@ -0,0 +1,147 @@ +package plugin + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/hashicorp/vault/sdk/logical" +) + +func setup() (context.Context, logical.Storage, string, *CheckOut) { + ctx := context.Background() + storage := &logical.InmemStorage{} + serviceAccountName := "becca@example.com" + checkOut := &CheckOut{ + BorrowerEntityID: "entity-id", + BorrowerClientToken: "client-token", + } + config := &configuration{ + PasswordConf: &passwordConf{ + Length: 14, + }, + } + entry, err := logical.StorageEntryJSON(configStorageKey, config) + if err != nil { + panic(err) + } + if err := storage.Put(ctx, entry); err != nil { + panic(err) + } + return ctx, storage, serviceAccountName, checkOut +} + +func TestCheckOutHandlerStorageLayer(t *testing.T) { + ctx, storage, serviceAccountName, testCheckOut := setup() + + storageHandler := &checkOutHandler{ + client: &fakeSecretsClient{}, + } + + // Service accounts must initially be checked in to the library + if err := storageHandler.CheckIn(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + + // If we try to check something out for the first time, it should succeed. + if err := storageHandler.CheckOut(ctx, storage, serviceAccountName, testCheckOut); err != nil { + t.Fatal(err) + } + + // We should have the testCheckOut in storage now. + storedCheckOut, err := storageHandler.LoadCheckOut(ctx, storage, serviceAccountName) + if err != nil { + t.Fatal(err) + } + if storedCheckOut == nil { + t.Fatal("storedCheckOut should not be nil") + } + if !reflect.DeepEqual(testCheckOut, storedCheckOut) { + t.Fatalf(fmt.Sprintf(`expected %+v to be equal to %+v`, testCheckOut, storedCheckOut)) + } + + // If we try to check something out that's already checked out, we should + // get a CurrentlyCheckedOutErr. + if err := storageHandler.CheckOut(ctx, storage, serviceAccountName, testCheckOut); err == nil { + t.Fatal("expected err but received none") + } else if err != errCheckedOut { + t.Fatalf("expected errCheckedOut, but received %s", err) + } + + // If we try to check something in, it should succeed. + if err := storageHandler.CheckIn(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + + // We should no longer have the testCheckOut in storage. + storedCheckOut, err = storageHandler.LoadCheckOut(ctx, storage, serviceAccountName) + if err != nil { + t.Fatal(err) + } + if !storedCheckOut.IsAvailable { + t.Fatal("storedCheckOut should be nil") + } + + // If we try to check it in again, it should have the same behavior. + if err := storageHandler.CheckIn(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + + // If we check it out again, it should succeed. + if err := storageHandler.CheckOut(ctx, storage, serviceAccountName, testCheckOut); err != nil { + t.Fatal(err) + } +} + +func TestPasswordHandlerInterfaceFulfillment(t *testing.T) { + ctx, storage, serviceAccountName, checkOut := setup() + + passwordHandler := &checkOutHandler{ + client: &fakeSecretsClient{}, + } + + // We must always start managing a service account by checking it in. + if err := passwordHandler.CheckIn(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + + // There should be no error during check-out. + if err := passwordHandler.CheckOut(ctx, storage, serviceAccountName, checkOut); err != nil { + t.Fatal(err) + } + + // The password should get rotated successfully during check-in. + origPassword, err := retrievePassword(ctx, storage, serviceAccountName) + if err != nil { + t.Fatal(err) + } + if err := passwordHandler.CheckIn(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + currPassword, err := retrievePassword(ctx, storage, serviceAccountName) + if err != nil { + t.Fatal(err) + } + if currPassword == "" || currPassword == origPassword { + t.Fatal("expected password, but received none") + } + + // There should be no error during delete and the password should be deleted. + if err := passwordHandler.Delete(ctx, storage, serviceAccountName); err != nil { + t.Fatal(err) + } + + currPassword, err = retrievePassword(ctx, storage, serviceAccountName) + if err != errNotFound { + t.Fatal("expected errNotFound") + } + + checkOut, err = passwordHandler.LoadCheckOut(ctx, storage, serviceAccountName) + if err != errNotFound { + t.Fatal("expected err not found") + } + if checkOut != nil { + t.Fatal("expected checkOut to be nil") + } +} diff --git a/plugin/path_checkout_sets.go b/plugin/path_checkout_sets.go new file mode 100644 index 0000000..f819038 --- /dev/null +++ b/plugin/path_checkout_sets.go @@ -0,0 +1,376 @@ +package plugin + +import ( + "context" + "fmt" + "time" + + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/locksutil" + "github.com/hashicorp/vault/sdk/helper/strutil" + "github.com/hashicorp/vault/sdk/logical" +) + +const libraryPrefix = "library/" + +type librarySet struct { + ServiceAccountNames []string `json:"service_account_names"` + TTL time.Duration `json:"ttl"` + MaxTTL time.Duration `json:"max_ttl"` + DisableCheckInEnforcement bool `json:"disable_check_in_enforcement"` +} + +// Validates ensures that a set meets our code assumptions that TTLs are set in +// a way that makes sense, and that there's at least one service account. +func (l *librarySet) Validate() error { + if len(l.ServiceAccountNames) < 1 { + return fmt.Errorf(`at least one service account must be configured`) + } + if l.MaxTTL > 0 { + if l.MaxTTL < l.TTL { + return fmt.Errorf(`max_ttl (%d seconds) may not be less than ttl (%d seconds)`, l.MaxTTL, l.TTL) + } + } + return nil +} + +func (b *backend) pathListSets() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + "?$", + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ListOperation: &framework.PathOperation{ + Callback: b.setListOperation, + }, + }, + HelpSynopsis: pathListSetsHelpSyn, + HelpDescription: pathListSetsHelpDesc, + } +} + +func (b *backend) setListOperation(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { + keys, err := req.Storage.List(ctx, libraryPrefix) + if err != nil { + return nil, err + } + return logical.ListResponse(keys), nil +} + +func (b *backend) pathSets() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + framework.GenericNameRegex("name"), + Fields: map[string]*framework.FieldSchema{ + "name": { + Type: framework.TypeLowerCaseString, + Description: "Name of the set.", + Required: true, + }, + "service_account_names": { + Type: framework.TypeCommaStringSlice, + Description: "The username/logon name for the service accounts with which this set will be associated.", + }, + "ttl": { + Type: framework.TypeDurationSecond, + Description: "In seconds, the amount of time a check-out should last. Defaults to 24 hours.", + Default: 24 * 60 * 60, // 24 hours + }, + "max_ttl": { + Type: framework.TypeDurationSecond, + Description: "In seconds, the max amount of time a check-out's renewals should last. Defaults to 24 hours.", + Default: 24 * 60 * 60, // 24 hours + }, + "disable_check_in_enforcement": { + Type: framework.TypeBool, + Description: "Disable the default behavior of requiring that check-ins are performed by the entity that checked them out.", + Default: false, + }, + }, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.CreateOperation: &framework.PathOperation{ + Callback: b.operationSetCreate, + Summary: "Create a library set.", + }, + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.operationSetUpdate, + Summary: "Update a library set.", + }, + logical.ReadOperation: &framework.PathOperation{ + Callback: b.operationSetRead, + Summary: "Read a library set.", + }, + logical.DeleteOperation: &framework.PathOperation{ + Callback: b.operationSetDelete, + Summary: "Delete a library set.", + }, + }, + ExistenceCheck: b.operationSetExistenceCheck, + HelpSynopsis: setHelpSynopsis, + HelpDescription: setHelpDescription, + } +} + +func (b *backend) operationSetExistenceCheck(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (bool, error) { + set, err := readSet(ctx, req.Storage, fieldData.Get("name").(string)) + if err != nil { + return false, err + } + return set != nil, nil +} + +func (b *backend) operationSetCreate(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + serviceAccountNames := fieldData.Get("service_account_names").([]string) + ttl := time.Duration(fieldData.Get("ttl").(int)) * time.Second + maxTTL := time.Duration(fieldData.Get("max_ttl").(int)) * time.Second + disableCheckInEnforcement := fieldData.Get("disable_check_in_enforcement").(bool) + + if len(serviceAccountNames) == 0 { + return logical.ErrorResponse(`"service_account_names" must be provided`), nil + } + + // Ensure these service accounts aren't already managed by another check-out set. + for _, serviceAccountName := range serviceAccountNames { + if _, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, serviceAccountName); err != nil { + if err == errNotFound { + // This is what we want to see. + continue + } + return nil, err + } + return logical.ErrorResponse(fmt.Sprintf("%q is already managed by another set", serviceAccountName)), nil + } + + set := &librarySet{ + ServiceAccountNames: serviceAccountNames, + TTL: ttl, + MaxTTL: maxTTL, + DisableCheckInEnforcement: disableCheckInEnforcement, + } + if err := set.Validate(); err != nil { + return logical.ErrorResponse(err.Error()), nil + } + for _, serviceAccountName := range serviceAccountNames { + if err := b.checkOutHandler.CheckIn(ctx, req.Storage, serviceAccountName); err != nil { + return nil, err + } + } + if err := storeSet(ctx, req.Storage, setName, set); err != nil { + return nil, err + } + return nil, nil +} + +func (b *backend) operationSetUpdate(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + newServiceAccountNamesRaw, newServiceAccountNamesSent := fieldData.GetOk("service_account_names") + var newServiceAccountNames []string + if newServiceAccountNamesSent { + newServiceAccountNames = newServiceAccountNamesRaw.([]string) + } + + ttlRaw, ttlSent := fieldData.GetOk("ttl") + if !ttlSent { + ttlRaw = fieldData.Schema["ttl"].Default + } + ttl := time.Duration(ttlRaw.(int)) * time.Second + + maxTTLRaw, maxTTLSent := fieldData.GetOk("max_ttl") + if !maxTTLSent { + maxTTLRaw = fieldData.Schema["max_ttl"].Default + } + maxTTL := time.Duration(maxTTLRaw.(int)) * time.Second + + disableCheckInEnforcementRaw, enforcementSent := fieldData.GetOk("disable_check_in_enforcement") + if !enforcementSent { + disableCheckInEnforcementRaw = false + } + disableCheckInEnforcement := disableCheckInEnforcementRaw.(bool) + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return logical.ErrorResponse(fmt.Sprintf(`%q doesn't exist`, setName)), nil + } + + var beingAdded []string + var beingDeleted []string + if newServiceAccountNamesSent { + + // For new service accounts we receive, before we check them in, ensure they're not in another set. + beingAdded = strutil.Difference(newServiceAccountNames, set.ServiceAccountNames, true) + for _, newServiceAccountName := range beingAdded { + if _, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, newServiceAccountName); err != nil { + if err == errNotFound { + // Great, this validates that it's not in use in another set. + continue + } + return nil, err + } + return logical.ErrorResponse(fmt.Sprintf("%q is already managed by another set", newServiceAccountName)), nil + } + + // For service accounts we won't be handling anymore, before we delete them, ensure they're not checked out. + beingDeleted = strutil.Difference(set.ServiceAccountNames, newServiceAccountNames, true) + for _, prevServiceAccountName := range beingDeleted { + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, prevServiceAccountName) + if err != nil { + if err == errNotFound { + // Nothing else to do here. + continue + } + return nil, err + } + if !checkOut.IsAvailable { + return logical.ErrorResponse(fmt.Sprintf(`"%s" can't be deleted because it is currently checked out'`, prevServiceAccountName)), nil + } + } + set.ServiceAccountNames = newServiceAccountNames + } + + if ttlSent { + set.TTL = ttl + } + if maxTTLSent { + set.MaxTTL = maxTTL + } + if enforcementSent { + set.DisableCheckInEnforcement = disableCheckInEnforcement + } + if err := set.Validate(); err != nil { + return logical.ErrorResponse(err.Error()), nil + } + + // Now that we know we can take all these actions, let's take them. + for _, newServiceAccountName := range beingAdded { + if err := b.checkOutHandler.CheckIn(ctx, req.Storage, newServiceAccountName); err != nil { + return nil, err + } + } + for _, prevServiceAccountName := range beingDeleted { + if err := b.checkOutHandler.Delete(ctx, req.Storage, prevServiceAccountName); err != nil { + return nil, err + } + } + if err := storeSet(ctx, req.Storage, setName, set); err != nil { + return nil, err + } + return nil, nil +} + +func (b *backend) operationSetRead(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.RLock() + defer lock.RUnlock() + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return nil, nil + } + return &logical.Response{ + Data: map[string]interface{}{ + "service_account_names": set.ServiceAccountNames, + "ttl": int64(set.TTL.Seconds()), + "max_ttl": int64(set.MaxTTL.Seconds()), + "disable_check_in_enforcement": set.DisableCheckInEnforcement, + }, + }, nil +} + +func (b *backend) operationSetDelete(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return nil, nil + } + // We need to remove all the items we'd stored for these service accounts. + for _, serviceAccountName := range set.ServiceAccountNames { + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, serviceAccountName) + if err != nil { + if err == errNotFound { + // Nothing else to do here. + continue + } + return nil, err + } + if !checkOut.IsAvailable { + return logical.ErrorResponse(fmt.Sprintf(`"%s" can't be deleted because it is currently checked out'`, serviceAccountName)), nil + } + } + for _, serviceAccountName := range set.ServiceAccountNames { + if err := b.checkOutHandler.Delete(ctx, req.Storage, serviceAccountName); err != nil { + return nil, err + } + } + if err := req.Storage.Delete(ctx, libraryPrefix+setName); err != nil { + return nil, err + } + return nil, nil +} + +// readSet is a helper method for reading a set from storage by name. +// It's intended to be used anywhere in the plugin. It may return nil, nil if +// a librarySet doesn't currently exist for a given setName. +func readSet(ctx context.Context, storage logical.Storage, setName string) (*librarySet, error) { + entry, err := storage.Get(ctx, libraryPrefix+setName) + if err != nil { + return nil, err + } + if entry == nil { + return nil, nil + } + set := &librarySet{} + if err := entry.DecodeJSON(set); err != nil { + return nil, err + } + return set, nil +} + +// storeSet stores a librarySet. +func storeSet(ctx context.Context, storage logical.Storage, setName string, set *librarySet) error { + entry, err := logical.StorageEntryJSON(libraryPrefix+setName, set) + if err != nil { + return err + } + return storage.Put(ctx, entry) +} + +const ( + setHelpSynopsis = ` +Manage sets to build a library of service accounts that can be checked out. +` + setHelpDescription = ` +This endpoint allows you to read, write, and delete individual sets that are used for checking out service accounts. +Deleting a set can only be performed if all of its service accounts are currently checked in. +` + pathListSetsHelpSyn = ` +List the name of each set currently stored. +` + pathListSetsHelpDesc = ` +To learn which service accounts are being managed by Vault, list the set names using +this endpoint. Then read any individual set by name to learn more. +` +) diff --git a/plugin/path_checkouts.go b/plugin/path_checkouts.go new file mode 100644 index 0000000..8d5478f --- /dev/null +++ b/plugin/path_checkouts.go @@ -0,0 +1,377 @@ +package plugin + +import ( + "context" + "fmt" + "time" + + metrics "github.com/armon/go-metrics" + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/locksutil" + "github.com/hashicorp/vault/sdk/logical" +) + +const secretAccessKeyType = "creds" + +func (b *backend) pathSetCheckOut() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + framework.GenericNameRegex("name") + "/check-out$", + Fields: map[string]*framework.FieldSchema{ + "name": { + Type: framework.TypeLowerCaseString, + Description: "Name of the set", + Required: true, + }, + "ttl": { + Type: framework.TypeDurationSecond, + Description: "The length of time before the check-out will expire, in seconds.", + }, + }, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.operationSetCheckOut, + Summary: "Check a service account out from the library.", + }, + }, + HelpSynopsis: `Check a service account out from the library.`, + } +} + +func (b *backend) operationSetCheckOut(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + ttlPeriodRaw, ttlPeriodSent := fieldData.GetOk("ttl") + if !ttlPeriodSent { + ttlPeriodRaw = 0 + } + requestedTTL := time.Duration(ttlPeriodRaw.(int)) * time.Second + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return logical.ErrorResponse(fmt.Sprintf(`%q doesn't exist`, setName)), nil + } + + // Prepare the check-out we'd like to execute. + ttl := set.TTL + if ttlPeriodSent { + switch { + case set.TTL <= 0 && requestedTTL > 0: + // The set's TTL is infinite and the caller requested a finite TTL. + ttl = requestedTTL + case set.TTL > 0 && requestedTTL < set.TTL: + // The set's TTL isn't infinite and the caller requested a shorter TTL. + ttl = requestedTTL + } + } + newCheckOut := &CheckOut{ + IsAvailable: false, + BorrowerEntityID: req.EntityID, + BorrowerClientToken: req.ClientToken, + } + + // Check out the first service account available. + for _, serviceAccountName := range set.ServiceAccountNames { + if err := b.checkOutHandler.CheckOut(ctx, req.Storage, serviceAccountName, newCheckOut); err != nil { + if err == errCheckedOut { + continue + } + return nil, err + } + password, err := retrievePassword(ctx, req.Storage, serviceAccountName) + if err != nil { + return nil, err + } + respData := map[string]interface{}{ + "service_account_name": serviceAccountName, + "password": password, + } + internalData := map[string]interface{}{ + "service_account_name": serviceAccountName, + "set_name": setName, + } + resp := b.Backend.Secret(secretAccessKeyType).Response(respData, internalData) + resp.Secret.Renewable = true + resp.Secret.TTL = ttl + resp.Secret.MaxTTL = set.MaxTTL + return resp, nil + } + + // If we arrived here, it's because we never had a hit for a service account that was available. + // In case of customer issues, we need to make this easy to see and diagnose. + b.Logger().Debug(fmt.Sprintf(`%q had no check-outs available`, setName)) + metrics.IncrCounter([]string{"active directory", "check-out", "unavailable", setName}, 1) + + return logical.RespondWithStatusCode(&logical.Response{ + Warnings: []string{"No service accounts available for check-out."}, + }, req, 400) +} + +func (b *backend) secretAccessKeys() *framework.Secret { + return &framework.Secret{ + Type: secretAccessKeyType, + Fields: map[string]*framework.FieldSchema{ + "service_account_name": { + Type: framework.TypeString, + Description: "Service account name", + }, + "password": { + Type: framework.TypeString, + Description: "Password", + }, + }, + Renew: b.renewCheckOut, + Revoke: b.endCheckOut, + } +} + +func (b *backend) renewCheckOut(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := req.Secret.InternalData["set_name"].(string) + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.RLock() + defer lock.RUnlock() + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return logical.ErrorResponse(fmt.Sprintf(`%q doesn't exist`, setName)), nil + } + + serviceAccountName := req.Secret.InternalData["service_account_name"].(string) + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, serviceAccountName) + if err != nil { + return nil, err + } + if checkOut.IsAvailable { + // It's possible that this renewal could be attempted after a check-in occurred either by this entity or by + // another user with access to the "manage check-ins" endpoint that forcibly checked it back in. + return logical.ErrorResponse(fmt.Sprintf("%s is already checked in, please call check-out to regain it", serviceAccountName)), nil + } + resp := &logical.Response{Secret: req.Secret} + resp.Secret.TTL = set.TTL + resp.Secret.MaxTTL = set.MaxTTL + return resp, nil +} + +func (b *backend) endCheckOut(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := req.Secret.InternalData["set_name"].(string) + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + serviceAccountName := req.Secret.InternalData["service_account_name"].(string) + if err := b.checkOutHandler.CheckIn(ctx, req.Storage, serviceAccountName); err != nil { + return nil, err + } + return nil, nil +} + +func (b *backend) pathSetCheckIn() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + framework.GenericNameRegex("name") + "/check-in$", + Fields: map[string]*framework.FieldSchema{ + "name": { + Type: framework.TypeLowerCaseString, + Description: "Name of the set.", + Required: true, + }, + "service_account_names": { + Type: framework.TypeCommaStringSlice, + Description: "The username/logon name for the service accounts to check in.", + }, + }, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.operationCheckIn(false), + Summary: "Check service accounts in to the library.", + }, + }, + HelpSynopsis: `Check service accounts in to the library.`, + } +} + +func (b *backend) pathSetManageCheckIn() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + "manage/" + framework.GenericNameRegex("name") + "/check-in$", + Fields: map[string]*framework.FieldSchema{ + "name": { + Type: framework.TypeLowerCaseString, + Description: "Name of the set.", + Required: true, + }, + "service_account_names": { + Type: framework.TypeCommaStringSlice, + Description: "The username/logon name for the service accounts to check in.", + }, + }, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.UpdateOperation: &framework.PathOperation{ + Callback: b.operationCheckIn(true), + Summary: "Check service accounts in to the library.", + }, + }, + HelpSynopsis: `Force checking service accounts in to the library.`, + } +} + +func (b *backend) operationCheckIn(overrideCheckInEnforcement bool) framework.OperationFunc { + return func(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.Lock() + defer lock.Unlock() + + serviceAccountNamesRaw, serviceAccountNamesSent := fieldData.GetOk("service_account_names") + var serviceAccountNames []string + if serviceAccountNamesSent { + serviceAccountNames = serviceAccountNamesRaw.([]string) + } + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return logical.ErrorResponse(fmt.Sprintf(`%q doesn't exist`, setName)), nil + } + + // If check-in enforcement is overridden or disabled at the set level, we should consider it disabled. + disableCheckInEnforcement := overrideCheckInEnforcement || set.DisableCheckInEnforcement + + // Track the service accounts we check in so we can include it in our response. + toCheckIn := make([]string, 0) + + // Build and validate a list of service account names that we will be checking in. + if len(serviceAccountNames) == 0 { + // It's okay if the caller doesn't tell us which service accounts they + // want to check in as long as they only have one checked out. + // We'll assume that's the one they want to check in. + for _, setServiceAccount := range set.ServiceAccountNames { + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, setServiceAccount) + if err != nil { + return nil, err + } + if checkOut.IsAvailable { + continue + } + if !disableCheckInEnforcement && !checkinAuthorized(req, checkOut) { + continue + } + toCheckIn = append(toCheckIn, setServiceAccount) + } + if len(toCheckIn) > 1 { + return logical.ErrorResponse(`when multiple service accounts are checked out, the "service_account_names" to check in must be provided`), nil + } + } else { + for _, serviceAccountName := range serviceAccountNames { + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, serviceAccountName) + if err != nil { + return nil, err + } + // First guard that they should be able to do anything at all. + if !checkOut.IsAvailable && !disableCheckInEnforcement && !checkinAuthorized(req, checkOut) { + return logical.ErrorResponse("%q can't be checked in because it wasn't checked out by the caller", serviceAccountName), nil + } + if checkOut.IsAvailable { + continue + } + toCheckIn = append(toCheckIn, serviceAccountName) + } + } + for _, serviceAccountName := range toCheckIn { + if err := b.checkOutHandler.CheckIn(ctx, req.Storage, serviceAccountName); err != nil { + return nil, err + } + } + return &logical.Response{ + Data: map[string]interface{}{ + "check_ins": toCheckIn, + }, + }, nil + } +} + +func (b *backend) pathSetStatus() *framework.Path { + return &framework.Path{ + Pattern: libraryPrefix + framework.GenericNameRegex("name") + "/status$", + Fields: map[string]*framework.FieldSchema{ + "name": { + Type: framework.TypeLowerCaseString, + Description: "Name of the set.", + Required: true, + }, + }, + Operations: map[logical.Operation]framework.OperationHandler{ + logical.ReadOperation: &framework.PathOperation{ + Callback: b.operationSetStatus, + Summary: "Check the status of the service accounts in a library set.", + }, + }, + HelpSynopsis: `Check the status of the service accounts in a library.`, + } +} + +func (b *backend) operationSetStatus(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { + setName := fieldData.Get("name").(string) + lock := locksutil.LockForKey(b.checkOutLocks, setName) + lock.RLock() + defer lock.RUnlock() + + set, err := readSet(ctx, req.Storage, setName) + if err != nil { + return nil, err + } + if set == nil { + return logical.ErrorResponse(fmt.Sprintf(`%q doesn't exist`, setName)), nil + } + respData := make(map[string]interface{}) + + for _, serviceAccountName := range set.ServiceAccountNames { + checkOut, err := b.checkOutHandler.LoadCheckOut(ctx, req.Storage, serviceAccountName) + if err != nil { + return nil, err + } + + status := map[string]interface{}{ + "available": checkOut.IsAvailable, + } + if checkOut.IsAvailable { + // We only omit all other fields if the checkout is currently available, + // because they're only relevant to accounts that aren't checked out. + respData[serviceAccountName] = status + continue + } + if checkOut.BorrowerClientToken != "" { + status["borrower_client_token"] = checkOut.BorrowerClientToken + } + if checkOut.BorrowerEntityID != "" { + status["borrower_entity_id"] = checkOut.BorrowerEntityID + } + respData[serviceAccountName] = status + } + return &logical.Response{ + Data: respData, + }, nil +} + +func checkinAuthorized(req *logical.Request, checkOut *CheckOut) bool { + if checkOut.BorrowerEntityID != "" && req.EntityID != "" { + if checkOut.BorrowerEntityID == req.EntityID { + return true + } + } + if checkOut.BorrowerClientToken != "" && req.ClientToken != "" { + if checkOut.BorrowerClientToken == req.ClientToken { + return true + } + } + return false +} diff --git a/plugin/path_checkouts_test.go b/plugin/path_checkouts_test.go new file mode 100644 index 0000000..2a3c807 --- /dev/null +++ b/plugin/path_checkouts_test.go @@ -0,0 +1,30 @@ +package plugin + +import ( + "testing" + + "github.com/hashicorp/vault/sdk/logical" +) + +func TestCheckInAuthorized(t *testing.T) { + can := checkinAuthorized(&logical.Request{EntityID: "foo"}, &CheckOut{BorrowerEntityID: "foo"}) + if !can { + t.Fatal("the entity that checked out the secret should be able to check it in") + } + can = checkinAuthorized(&logical.Request{ClientToken: "foo"}, &CheckOut{BorrowerClientToken: "foo"}) + if !can { + t.Fatal("the client token that checked out the secret should be able to check it in") + } + can = checkinAuthorized(&logical.Request{EntityID: "fizz"}, &CheckOut{BorrowerEntityID: "buzz"}) + if can { + t.Fatal("other entities shouldn't be able to perform check-ins") + } + can = checkinAuthorized(&logical.Request{ClientToken: "fizz"}, &CheckOut{BorrowerClientToken: "buzz"}) + if can { + t.Fatal("other tokens shouldn't be able to perform check-ins") + } + can = checkinAuthorized(&logical.Request{}, &CheckOut{}) + if can { + t.Fatal("when insufficient auth info is provided, check-in should not be allowed") + } +} diff --git a/plugin/path_config.go b/plugin/path_config.go index a79a688..a5345a8 100644 --- a/plugin/path_config.go +++ b/plugin/path_config.go @@ -24,7 +24,7 @@ const ( defaultTLSVersion = "tls12" ) -func (b *backend) readConfig(ctx context.Context, storage logical.Storage) (*configuration, error) { +func readConfig(ctx context.Context, storage logical.Storage) (*configuration, error) { entry, err := storage.Get(ctx, configStorageKey) if err != nil { return nil, err @@ -145,7 +145,7 @@ func (b *backend) configUpdateOperation(ctx context.Context, req *logical.Reques } func (b *backend) configReadOperation(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { - config, err := b.readConfig(ctx, req.Storage) + config, err := readConfig(ctx, req.Storage) if err != nil { return nil, err } diff --git a/plugin/path_config_test.go b/plugin/path_config_test.go index 9f42c83..01de094 100644 --- a/plugin/path_config_test.go +++ b/plugin/path_config_test.go @@ -16,7 +16,7 @@ var ( func TestCacheReader(t *testing.T) { // we should start with no config - config, err := testBackend.readConfig(ctx, storage) + config, err := readConfig(ctx, storage) if err != nil { t.Fatal(err) } @@ -47,7 +47,7 @@ func TestCacheReader(t *testing.T) { } // now that we've updated the config, we should be able to configReadOperation it - config, err = testBackend.readConfig(ctx, storage) + config, err = readConfig(ctx, storage) if err != nil { t.Fatal(err) } @@ -80,7 +80,7 @@ func TestCacheReader(t *testing.T) { } // now that we've deleted the config, it should be unset again - config, err = testBackend.readConfig(ctx, storage) + config, err = readConfig(ctx, storage) if err != nil { t.Fatal(err) } diff --git a/plugin/path_creds.go b/plugin/path_creds.go index bbb2abe..c2a192b 100644 --- a/plugin/path_creds.go +++ b/plugin/path_creds.go @@ -59,7 +59,7 @@ func (b *backend) pathCreds() *framework.Path { func (b *backend) credReadOperation(ctx context.Context, req *logical.Request, fieldData *framework.FieldData) (*logical.Response, error) { cred := make(map[string]interface{}) - engineConf, err := b.readConfig(ctx, req.Storage) + engineConf, err := readConfig(ctx, req.Storage) if err != nil { return nil, err } diff --git a/plugin/path_roles.go b/plugin/path_roles.go index 594888f..2f92f0b 100644 --- a/plugin/path_roles.go +++ b/plugin/path_roles.go @@ -89,7 +89,7 @@ func (b *backend) readRole(ctx context.Context, storage logical.Storage, roleNam } // Always check when ActiveDirectory shows the password as last set on the fly. - engineConf, err := b.readConfig(ctx, storage) + engineConf, err := readConfig(ctx, storage) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func (b *backend) roleUpdateOperation(ctx context.Context, req *logical.Request, // Get everything we need to construct the role. roleName := fieldData.Get("name").(string) - engineConf, err := b.readConfig(ctx, req.Storage) + engineConf, err := readConfig(ctx, req.Storage) if err != nil { return nil, err } diff --git a/plugin/path_rotate_root_creds.go b/plugin/path_rotate_root_creds.go index 2853f62..a7b4448 100644 --- a/plugin/path_rotate_root_creds.go +++ b/plugin/path_rotate_root_creds.go @@ -25,7 +25,7 @@ func (b *backend) pathRotateCredentials() *framework.Path { } func (b *backend) pathRotateCredentialsUpdate(ctx context.Context, req *logical.Request, _ *framework.FieldData) (*logical.Response, error) { - engineConf, err := b.readConfig(ctx, req.Storage) + engineConf, err := readConfig(ctx, req.Storage) if err != nil { return nil, err } diff --git a/vendor/github.com/armon/go-metrics/.gitignore b/vendor/github.com/armon/go-metrics/.gitignore new file mode 100644 index 0000000..8c03ec1 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +/metrics.out diff --git a/vendor/github.com/armon/go-metrics/LICENSE b/vendor/github.com/armon/go-metrics/LICENSE new file mode 100644 index 0000000..106569e --- /dev/null +++ b/vendor/github.com/armon/go-metrics/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/armon/go-metrics/README.md b/vendor/github.com/armon/go-metrics/README.md new file mode 100644 index 0000000..aa73348 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/README.md @@ -0,0 +1,91 @@ +go-metrics +========== + +This library provides a `metrics` package which can be used to instrument code, +expose application metrics, and profile runtime performance in a flexible manner. + +Current API: [![GoDoc](https://godoc.org/github.com/armon/go-metrics?status.svg)](https://godoc.org/github.com/armon/go-metrics) + +Sinks +----- + +The `metrics` package makes use of a `MetricSink` interface to support delivery +to any type of backend. Currently the following sinks are provided: + +* StatsiteSink : Sinks to a [statsite](https://github.com/armon/statsite/) instance (TCP) +* StatsdSink: Sinks to a [StatsD](https://github.com/etsy/statsd/) / statsite instance (UDP) +* PrometheusSink: Sinks to a [Prometheus](http://prometheus.io/) metrics endpoint (exposed via HTTP for scrapes) +* InmemSink : Provides in-memory aggregation, can be used to export stats +* FanoutSink : Sinks to multiple sinks. Enables writing to multiple statsite instances for example. +* BlackholeSink : Sinks to nowhere + +In addition to the sinks, the `InmemSignal` can be used to catch a signal, +and dump a formatted output of recent metrics. For example, when a process gets +a SIGUSR1, it can dump to stderr recent performance metrics for debugging. + +Labels +------ + +Most metrics do have an equivalent ending with `WithLabels`, such methods +allow to push metrics with labels and use some features of underlying Sinks +(ex: translated into Prometheus labels). + +Since some of these labels may increase greatly cardinality of metrics, the +library allow to filter labels using a blacklist/whitelist filtering system +which is global to all metrics. + +* If `Config.AllowedLabels` is not nil, then only labels specified in this value will be sent to underlying Sink, otherwise, all labels are sent by default. +* If `Config.BlockedLabels` is not nil, any label specified in this value will not be sent to underlying Sinks. + +By default, both `Config.AllowedLabels` and `Config.BlockedLabels` are nil, meaning that +no tags are filetered at all, but it allow to a user to globally block some tags with high +cardinality at application level. + +Examples +-------- + +Here is an example of using the package: + +```go +func SlowMethod() { + // Profiling the runtime of a method + defer metrics.MeasureSince([]string{"SlowMethod"}, time.Now()) +} + +// Configure a statsite sink as the global metrics sink +sink, _ := metrics.NewStatsiteSink("statsite:8125") +metrics.NewGlobal(metrics.DefaultConfig("service-name"), sink) + +// Emit a Key/Value pair +metrics.EmitKey([]string{"questions", "meaning of life"}, 42) +``` + +Here is an example of setting up a signal handler: + +```go +// Setup the inmem sink and signal handler +inm := metrics.NewInmemSink(10*time.Second, time.Minute) +sig := metrics.DefaultInmemSignal(inm) +metrics.NewGlobal(metrics.DefaultConfig("service-name"), inm) + +// Run some code +inm.SetGauge([]string{"foo"}, 42) +inm.EmitKey([]string{"bar"}, 30) + +inm.IncrCounter([]string{"baz"}, 42) +inm.IncrCounter([]string{"baz"}, 1) +inm.IncrCounter([]string{"baz"}, 80) + +inm.AddSample([]string{"method", "wow"}, 42) +inm.AddSample([]string{"method", "wow"}, 100) +inm.AddSample([]string{"method", "wow"}, 22) + +.... +``` + +When a signal comes in, output like the following will be dumped to stderr: + + [2014-01-28 14:57:33.04 -0800 PST][G] 'foo': 42.000 + [2014-01-28 14:57:33.04 -0800 PST][P] 'bar': 30.000 + [2014-01-28 14:57:33.04 -0800 PST][C] 'baz': Count: 3 Min: 1.000 Mean: 41.000 Max: 80.000 Stddev: 39.509 + [2014-01-28 14:57:33.04 -0800 PST][S] 'method.wow': Count: 3 Min: 22.000 Mean: 54.667 Max: 100.000 Stddev: 40.513 \ No newline at end of file diff --git a/vendor/github.com/armon/go-metrics/const_unix.go b/vendor/github.com/armon/go-metrics/const_unix.go new file mode 100644 index 0000000..31098dd --- /dev/null +++ b/vendor/github.com/armon/go-metrics/const_unix.go @@ -0,0 +1,12 @@ +// +build !windows + +package metrics + +import ( + "syscall" +) + +const ( + // DefaultSignal is used with DefaultInmemSignal + DefaultSignal = syscall.SIGUSR1 +) diff --git a/vendor/github.com/armon/go-metrics/const_windows.go b/vendor/github.com/armon/go-metrics/const_windows.go new file mode 100644 index 0000000..38136af --- /dev/null +++ b/vendor/github.com/armon/go-metrics/const_windows.go @@ -0,0 +1,13 @@ +// +build windows + +package metrics + +import ( + "syscall" +) + +const ( + // DefaultSignal is used with DefaultInmemSignal + // Windows has no SIGUSR1, use SIGBREAK + DefaultSignal = syscall.Signal(21) +) diff --git a/vendor/github.com/armon/go-metrics/inmem.go b/vendor/github.com/armon/go-metrics/inmem.go new file mode 100644 index 0000000..4e2d6a7 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/inmem.go @@ -0,0 +1,348 @@ +package metrics + +import ( + "bytes" + "fmt" + "math" + "net/url" + "strings" + "sync" + "time" +) + +// InmemSink provides a MetricSink that does in-memory aggregation +// without sending metrics over a network. It can be embedded within +// an application to provide profiling information. +type InmemSink struct { + // How long is each aggregation interval + interval time.Duration + + // Retain controls how many metrics interval we keep + retain time.Duration + + // maxIntervals is the maximum length of intervals. + // It is retain / interval. + maxIntervals int + + // intervals is a slice of the retained intervals + intervals []*IntervalMetrics + intervalLock sync.RWMutex + + rateDenom float64 +} + +// IntervalMetrics stores the aggregated metrics +// for a specific interval +type IntervalMetrics struct { + sync.RWMutex + + // The start time of the interval + Interval time.Time + + // Gauges maps the key to the last set value + Gauges map[string]GaugeValue + + // Points maps the string to the list of emitted values + // from EmitKey + Points map[string][]float32 + + // Counters maps the string key to a sum of the counter + // values + Counters map[string]SampledValue + + // Samples maps the key to an AggregateSample, + // which has the rolled up view of a sample + Samples map[string]SampledValue +} + +// NewIntervalMetrics creates a new IntervalMetrics for a given interval +func NewIntervalMetrics(intv time.Time) *IntervalMetrics { + return &IntervalMetrics{ + Interval: intv, + Gauges: make(map[string]GaugeValue), + Points: make(map[string][]float32), + Counters: make(map[string]SampledValue), + Samples: make(map[string]SampledValue), + } +} + +// AggregateSample is used to hold aggregate metrics +// about a sample +type AggregateSample struct { + Count int // The count of emitted pairs + Rate float64 // The values rate per time unit (usually 1 second) + Sum float64 // The sum of values + SumSq float64 `json:"-"` // The sum of squared values + Min float64 // Minimum value + Max float64 // Maximum value + LastUpdated time.Time `json:"-"` // When value was last updated +} + +// Computes a Stddev of the values +func (a *AggregateSample) Stddev() float64 { + num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2) + div := float64(a.Count * (a.Count - 1)) + if div == 0 { + return 0 + } + return math.Sqrt(num / div) +} + +// Computes a mean of the values +func (a *AggregateSample) Mean() float64 { + if a.Count == 0 { + return 0 + } + return a.Sum / float64(a.Count) +} + +// Ingest is used to update a sample +func (a *AggregateSample) Ingest(v float64, rateDenom float64) { + a.Count++ + a.Sum += v + a.SumSq += (v * v) + if v < a.Min || a.Count == 1 { + a.Min = v + } + if v > a.Max || a.Count == 1 { + a.Max = v + } + a.Rate = float64(a.Sum) / rateDenom + a.LastUpdated = time.Now() +} + +func (a *AggregateSample) String() string { + if a.Count == 0 { + return "Count: 0" + } else if a.Stddev() == 0 { + return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated) + } else { + return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s", + a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated) + } +} + +// NewInmemSinkFromURL creates an InmemSink from a URL. It is used +// (and tested) from NewMetricSinkFromURL. +func NewInmemSinkFromURL(u *url.URL) (MetricSink, error) { + params := u.Query() + + interval, err := time.ParseDuration(params.Get("interval")) + if err != nil { + return nil, fmt.Errorf("Bad 'interval' param: %s", err) + } + + retain, err := time.ParseDuration(params.Get("retain")) + if err != nil { + return nil, fmt.Errorf("Bad 'retain' param: %s", err) + } + + return NewInmemSink(interval, retain), nil +} + +// NewInmemSink is used to construct a new in-memory sink. +// Uses an aggregation interval and maximum retention period. +func NewInmemSink(interval, retain time.Duration) *InmemSink { + rateTimeUnit := time.Second + i := &InmemSink{ + interval: interval, + retain: retain, + maxIntervals: int(retain / interval), + rateDenom: float64(interval.Nanoseconds()) / float64(rateTimeUnit.Nanoseconds()), + } + i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals) + return i +} + +func (i *InmemSink) SetGauge(key []string, val float32) { + i.SetGaugeWithLabels(key, val, nil) +} + +func (i *InmemSink) SetGaugeWithLabels(key []string, val float32, labels []Label) { + k, name := i.flattenKeyLabels(key, labels) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + intv.Gauges[k] = GaugeValue{Name: name, Value: val, Labels: labels} +} + +func (i *InmemSink) EmitKey(key []string, val float32) { + k := i.flattenKey(key) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + vals := intv.Points[k] + intv.Points[k] = append(vals, val) +} + +func (i *InmemSink) IncrCounter(key []string, val float32) { + i.IncrCounterWithLabels(key, val, nil) +} + +func (i *InmemSink) IncrCounterWithLabels(key []string, val float32, labels []Label) { + k, name := i.flattenKeyLabels(key, labels) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + + agg, ok := intv.Counters[k] + if !ok { + agg = SampledValue{ + Name: name, + AggregateSample: &AggregateSample{}, + Labels: labels, + } + intv.Counters[k] = agg + } + agg.Ingest(float64(val), i.rateDenom) +} + +func (i *InmemSink) AddSample(key []string, val float32) { + i.AddSampleWithLabels(key, val, nil) +} + +func (i *InmemSink) AddSampleWithLabels(key []string, val float32, labels []Label) { + k, name := i.flattenKeyLabels(key, labels) + intv := i.getInterval() + + intv.Lock() + defer intv.Unlock() + + agg, ok := intv.Samples[k] + if !ok { + agg = SampledValue{ + Name: name, + AggregateSample: &AggregateSample{}, + Labels: labels, + } + intv.Samples[k] = agg + } + agg.Ingest(float64(val), i.rateDenom) +} + +// Data is used to retrieve all the aggregated metrics +// Intervals may be in use, and a read lock should be acquired +func (i *InmemSink) Data() []*IntervalMetrics { + // Get the current interval, forces creation + i.getInterval() + + i.intervalLock.RLock() + defer i.intervalLock.RUnlock() + + n := len(i.intervals) + intervals := make([]*IntervalMetrics, n) + + copy(intervals[:n-1], i.intervals[:n-1]) + current := i.intervals[n-1] + + // make its own copy for current interval + intervals[n-1] = &IntervalMetrics{} + copyCurrent := intervals[n-1] + current.RLock() + *copyCurrent = *current + + copyCurrent.Gauges = make(map[string]GaugeValue, len(current.Gauges)) + for k, v := range current.Gauges { + copyCurrent.Gauges[k] = v + } + // saved values will be not change, just copy its link + copyCurrent.Points = make(map[string][]float32, len(current.Points)) + for k, v := range current.Points { + copyCurrent.Points[k] = v + } + copyCurrent.Counters = make(map[string]SampledValue, len(current.Counters)) + for k, v := range current.Counters { + copyCurrent.Counters[k] = v + } + copyCurrent.Samples = make(map[string]SampledValue, len(current.Samples)) + for k, v := range current.Samples { + copyCurrent.Samples[k] = v + } + current.RUnlock() + + return intervals +} + +func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics { + i.intervalLock.RLock() + defer i.intervalLock.RUnlock() + + n := len(i.intervals) + if n > 0 && i.intervals[n-1].Interval == intv { + return i.intervals[n-1] + } + return nil +} + +func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics { + i.intervalLock.Lock() + defer i.intervalLock.Unlock() + + // Check for an existing interval + n := len(i.intervals) + if n > 0 && i.intervals[n-1].Interval == intv { + return i.intervals[n-1] + } + + // Add the current interval + current := NewIntervalMetrics(intv) + i.intervals = append(i.intervals, current) + n++ + + // Truncate the intervals if they are too long + if n >= i.maxIntervals { + copy(i.intervals[0:], i.intervals[n-i.maxIntervals:]) + i.intervals = i.intervals[:i.maxIntervals] + } + return current +} + +// getInterval returns the current interval to write to +func (i *InmemSink) getInterval() *IntervalMetrics { + intv := time.Now().Truncate(i.interval) + if m := i.getExistingInterval(intv); m != nil { + return m + } + return i.createInterval(intv) +} + +// Flattens the key for formatting, removes spaces +func (i *InmemSink) flattenKey(parts []string) string { + buf := &bytes.Buffer{} + replacer := strings.NewReplacer(" ", "_") + + if len(parts) > 0 { + replacer.WriteString(buf, parts[0]) + } + for _, part := range parts[1:] { + replacer.WriteString(buf, ".") + replacer.WriteString(buf, part) + } + + return buf.String() +} + +// Flattens the key for formatting along with its labels, removes spaces +func (i *InmemSink) flattenKeyLabels(parts []string, labels []Label) (string, string) { + buf := &bytes.Buffer{} + replacer := strings.NewReplacer(" ", "_") + + if len(parts) > 0 { + replacer.WriteString(buf, parts[0]) + } + for _, part := range parts[1:] { + replacer.WriteString(buf, ".") + replacer.WriteString(buf, part) + } + + key := buf.String() + + for _, label := range labels { + replacer.WriteString(buf, fmt.Sprintf(";%s=%s", label.Name, label.Value)) + } + + return buf.String(), key +} diff --git a/vendor/github.com/armon/go-metrics/inmem_endpoint.go b/vendor/github.com/armon/go-metrics/inmem_endpoint.go new file mode 100644 index 0000000..504f1b3 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/inmem_endpoint.go @@ -0,0 +1,118 @@ +package metrics + +import ( + "fmt" + "net/http" + "sort" + "time" +) + +// MetricsSummary holds a roll-up of metrics info for a given interval +type MetricsSummary struct { + Timestamp string + Gauges []GaugeValue + Points []PointValue + Counters []SampledValue + Samples []SampledValue +} + +type GaugeValue struct { + Name string + Hash string `json:"-"` + Value float32 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +type PointValue struct { + Name string + Points []float32 +} + +type SampledValue struct { + Name string + Hash string `json:"-"` + *AggregateSample + Mean float64 + Stddev float64 + + Labels []Label `json:"-"` + DisplayLabels map[string]string `json:"Labels"` +} + +// DisplayMetrics returns a summary of the metrics from the most recent finished interval. +func (i *InmemSink) DisplayMetrics(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + data := i.Data() + + var interval *IntervalMetrics + n := len(data) + switch { + case n == 0: + return nil, fmt.Errorf("no metric intervals have been initialized yet") + case n == 1: + // Show the current interval if it's all we have + interval = i.intervals[0] + default: + // Show the most recent finished interval if we have one + interval = i.intervals[n-2] + } + + summary := MetricsSummary{ + Timestamp: interval.Interval.Round(time.Second).UTC().String(), + Gauges: make([]GaugeValue, 0, len(interval.Gauges)), + Points: make([]PointValue, 0, len(interval.Points)), + } + + // Format and sort the output of each metric type, so it gets displayed in a + // deterministic order. + for name, points := range interval.Points { + summary.Points = append(summary.Points, PointValue{name, points}) + } + sort.Slice(summary.Points, func(i, j int) bool { + return summary.Points[i].Name < summary.Points[j].Name + }) + + for hash, value := range interval.Gauges { + value.Hash = hash + value.DisplayLabels = make(map[string]string) + for _, label := range value.Labels { + value.DisplayLabels[label.Name] = label.Value + } + value.Labels = nil + + summary.Gauges = append(summary.Gauges, value) + } + sort.Slice(summary.Gauges, func(i, j int) bool { + return summary.Gauges[i].Hash < summary.Gauges[j].Hash + }) + + summary.Counters = formatSamples(interval.Counters) + summary.Samples = formatSamples(interval.Samples) + + return summary, nil +} + +func formatSamples(source map[string]SampledValue) []SampledValue { + output := make([]SampledValue, 0, len(source)) + for hash, sample := range source { + displayLabels := make(map[string]string) + for _, label := range sample.Labels { + displayLabels[label.Name] = label.Value + } + + output = append(output, SampledValue{ + Name: sample.Name, + Hash: hash, + AggregateSample: sample.AggregateSample, + Mean: sample.AggregateSample.Mean(), + Stddev: sample.AggregateSample.Stddev(), + DisplayLabels: displayLabels, + }) + } + sort.Slice(output, func(i, j int) bool { + return output[i].Hash < output[j].Hash + }) + + return output +} diff --git a/vendor/github.com/armon/go-metrics/inmem_signal.go b/vendor/github.com/armon/go-metrics/inmem_signal.go new file mode 100644 index 0000000..0937f4a --- /dev/null +++ b/vendor/github.com/armon/go-metrics/inmem_signal.go @@ -0,0 +1,117 @@ +package metrics + +import ( + "bytes" + "fmt" + "io" + "os" + "os/signal" + "strings" + "sync" + "syscall" +) + +// InmemSignal is used to listen for a given signal, and when received, +// to dump the current metrics from the InmemSink to an io.Writer +type InmemSignal struct { + signal syscall.Signal + inm *InmemSink + w io.Writer + sigCh chan os.Signal + + stop bool + stopCh chan struct{} + stopLock sync.Mutex +} + +// NewInmemSignal creates a new InmemSignal which listens for a given signal, +// and dumps the current metrics out to a writer +func NewInmemSignal(inmem *InmemSink, sig syscall.Signal, w io.Writer) *InmemSignal { + i := &InmemSignal{ + signal: sig, + inm: inmem, + w: w, + sigCh: make(chan os.Signal, 1), + stopCh: make(chan struct{}), + } + signal.Notify(i.sigCh, sig) + go i.run() + return i +} + +// DefaultInmemSignal returns a new InmemSignal that responds to SIGUSR1 +// and writes output to stderr. Windows uses SIGBREAK +func DefaultInmemSignal(inmem *InmemSink) *InmemSignal { + return NewInmemSignal(inmem, DefaultSignal, os.Stderr) +} + +// Stop is used to stop the InmemSignal from listening +func (i *InmemSignal) Stop() { + i.stopLock.Lock() + defer i.stopLock.Unlock() + + if i.stop { + return + } + i.stop = true + close(i.stopCh) + signal.Stop(i.sigCh) +} + +// run is a long running routine that handles signals +func (i *InmemSignal) run() { + for { + select { + case <-i.sigCh: + i.dumpStats() + case <-i.stopCh: + return + } + } +} + +// dumpStats is used to dump the data to output writer +func (i *InmemSignal) dumpStats() { + buf := bytes.NewBuffer(nil) + + data := i.inm.Data() + // Skip the last period which is still being aggregated + for j := 0; j < len(data)-1; j++ { + intv := data[j] + intv.RLock() + for _, val := range intv.Gauges { + name := i.flattenLabels(val.Name, val.Labels) + fmt.Fprintf(buf, "[%v][G] '%s': %0.3f\n", intv.Interval, name, val.Value) + } + for name, vals := range intv.Points { + for _, val := range vals { + fmt.Fprintf(buf, "[%v][P] '%s': %0.3f\n", intv.Interval, name, val) + } + } + for _, agg := range intv.Counters { + name := i.flattenLabels(agg.Name, agg.Labels) + fmt.Fprintf(buf, "[%v][C] '%s': %s\n", intv.Interval, name, agg.AggregateSample) + } + for _, agg := range intv.Samples { + name := i.flattenLabels(agg.Name, agg.Labels) + fmt.Fprintf(buf, "[%v][S] '%s': %s\n", intv.Interval, name, agg.AggregateSample) + } + intv.RUnlock() + } + + // Write out the bytes + i.w.Write(buf.Bytes()) +} + +// Flattens the key for formatting along with its labels, removes spaces +func (i *InmemSignal) flattenLabels(name string, labels []Label) string { + buf := bytes.NewBufferString(name) + replacer := strings.NewReplacer(" ", "_", ":", "_") + + for _, label := range labels { + replacer.WriteString(buf, ".") + replacer.WriteString(buf, label.Value) + } + + return buf.String() +} diff --git a/vendor/github.com/armon/go-metrics/metrics.go b/vendor/github.com/armon/go-metrics/metrics.go new file mode 100644 index 0000000..cf9def7 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/metrics.go @@ -0,0 +1,278 @@ +package metrics + +import ( + "runtime" + "strings" + "time" + + "github.com/hashicorp/go-immutable-radix" +) + +type Label struct { + Name string + Value string +} + +func (m *Metrics) SetGauge(key []string, val float32) { + m.SetGaugeWithLabels(key, val, nil) +} + +func (m *Metrics) SetGaugeWithLabels(key []string, val float32, labels []Label) { + if m.HostName != "" { + if m.EnableHostnameLabel { + labels = append(labels, Label{"host", m.HostName}) + } else if m.EnableHostname { + key = insert(0, m.HostName, key) + } + } + if m.EnableTypePrefix { + key = insert(0, "gauge", key) + } + if m.ServiceName != "" { + if m.EnableServiceLabel { + labels = append(labels, Label{"service", m.ServiceName}) + } else { + key = insert(0, m.ServiceName, key) + } + } + allowed, labelsFiltered := m.allowMetric(key, labels) + if !allowed { + return + } + m.sink.SetGaugeWithLabels(key, val, labelsFiltered) +} + +func (m *Metrics) EmitKey(key []string, val float32) { + if m.EnableTypePrefix { + key = insert(0, "kv", key) + } + if m.ServiceName != "" { + key = insert(0, m.ServiceName, key) + } + allowed, _ := m.allowMetric(key, nil) + if !allowed { + return + } + m.sink.EmitKey(key, val) +} + +func (m *Metrics) IncrCounter(key []string, val float32) { + m.IncrCounterWithLabels(key, val, nil) +} + +func (m *Metrics) IncrCounterWithLabels(key []string, val float32, labels []Label) { + if m.HostName != "" && m.EnableHostnameLabel { + labels = append(labels, Label{"host", m.HostName}) + } + if m.EnableTypePrefix { + key = insert(0, "counter", key) + } + if m.ServiceName != "" { + if m.EnableServiceLabel { + labels = append(labels, Label{"service", m.ServiceName}) + } else { + key = insert(0, m.ServiceName, key) + } + } + allowed, labelsFiltered := m.allowMetric(key, labels) + if !allowed { + return + } + m.sink.IncrCounterWithLabels(key, val, labelsFiltered) +} + +func (m *Metrics) AddSample(key []string, val float32) { + m.AddSampleWithLabels(key, val, nil) +} + +func (m *Metrics) AddSampleWithLabels(key []string, val float32, labels []Label) { + if m.HostName != "" && m.EnableHostnameLabel { + labels = append(labels, Label{"host", m.HostName}) + } + if m.EnableTypePrefix { + key = insert(0, "sample", key) + } + if m.ServiceName != "" { + if m.EnableServiceLabel { + labels = append(labels, Label{"service", m.ServiceName}) + } else { + key = insert(0, m.ServiceName, key) + } + } + allowed, labelsFiltered := m.allowMetric(key, labels) + if !allowed { + return + } + m.sink.AddSampleWithLabels(key, val, labelsFiltered) +} + +func (m *Metrics) MeasureSince(key []string, start time.Time) { + m.MeasureSinceWithLabels(key, start, nil) +} + +func (m *Metrics) MeasureSinceWithLabels(key []string, start time.Time, labels []Label) { + if m.HostName != "" && m.EnableHostnameLabel { + labels = append(labels, Label{"host", m.HostName}) + } + if m.EnableTypePrefix { + key = insert(0, "timer", key) + } + if m.ServiceName != "" { + if m.EnableServiceLabel { + labels = append(labels, Label{"service", m.ServiceName}) + } else { + key = insert(0, m.ServiceName, key) + } + } + allowed, labelsFiltered := m.allowMetric(key, labels) + if !allowed { + return + } + now := time.Now() + elapsed := now.Sub(start) + msec := float32(elapsed.Nanoseconds()) / float32(m.TimerGranularity) + m.sink.AddSampleWithLabels(key, msec, labelsFiltered) +} + +// UpdateFilter overwrites the existing filter with the given rules. +func (m *Metrics) UpdateFilter(allow, block []string) { + m.UpdateFilterAndLabels(allow, block, m.AllowedLabels, m.BlockedLabels) +} + +// UpdateFilterAndLabels overwrites the existing filter with the given rules. +func (m *Metrics) UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels []string) { + m.filterLock.Lock() + defer m.filterLock.Unlock() + + m.AllowedPrefixes = allow + m.BlockedPrefixes = block + + if allowedLabels == nil { + // Having a white list means we take only elements from it + m.allowedLabels = nil + } else { + m.allowedLabels = make(map[string]bool) + for _, v := range allowedLabels { + m.allowedLabels[v] = true + } + } + m.blockedLabels = make(map[string]bool) + for _, v := range blockedLabels { + m.blockedLabels[v] = true + } + m.AllowedLabels = allowedLabels + m.BlockedLabels = blockedLabels + + m.filter = iradix.New() + for _, prefix := range m.AllowedPrefixes { + m.filter, _, _ = m.filter.Insert([]byte(prefix), true) + } + for _, prefix := range m.BlockedPrefixes { + m.filter, _, _ = m.filter.Insert([]byte(prefix), false) + } +} + +// labelIsAllowed return true if a should be included in metric +// the caller should lock m.filterLock while calling this method +func (m *Metrics) labelIsAllowed(label *Label) bool { + labelName := (*label).Name + if m.blockedLabels != nil { + _, ok := m.blockedLabels[labelName] + if ok { + // If present, let's remove this label + return false + } + } + if m.allowedLabels != nil { + _, ok := m.allowedLabels[labelName] + return ok + } + // Allow by default + return true +} + +// filterLabels return only allowed labels +// the caller should lock m.filterLock while calling this method +func (m *Metrics) filterLabels(labels []Label) []Label { + if labels == nil { + return nil + } + toReturn := labels[:0] + for _, label := range labels { + if m.labelIsAllowed(&label) { + toReturn = append(toReturn, label) + } + } + return toReturn +} + +// Returns whether the metric should be allowed based on configured prefix filters +// Also return the applicable labels +func (m *Metrics) allowMetric(key []string, labels []Label) (bool, []Label) { + m.filterLock.RLock() + defer m.filterLock.RUnlock() + + if m.filter == nil || m.filter.Len() == 0 { + return m.Config.FilterDefault, m.filterLabels(labels) + } + + _, allowed, ok := m.filter.Root().LongestPrefix([]byte(strings.Join(key, "."))) + if !ok { + return m.Config.FilterDefault, m.filterLabels(labels) + } + + return allowed.(bool), m.filterLabels(labels) +} + +// Periodically collects runtime stats to publish +func (m *Metrics) collectStats() { + for { + time.Sleep(m.ProfileInterval) + m.emitRuntimeStats() + } +} + +// Emits various runtime statsitics +func (m *Metrics) emitRuntimeStats() { + // Export number of Goroutines + numRoutines := runtime.NumGoroutine() + m.SetGauge([]string{"runtime", "num_goroutines"}, float32(numRoutines)) + + // Export memory stats + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + m.SetGauge([]string{"runtime", "alloc_bytes"}, float32(stats.Alloc)) + m.SetGauge([]string{"runtime", "sys_bytes"}, float32(stats.Sys)) + m.SetGauge([]string{"runtime", "malloc_count"}, float32(stats.Mallocs)) + m.SetGauge([]string{"runtime", "free_count"}, float32(stats.Frees)) + m.SetGauge([]string{"runtime", "heap_objects"}, float32(stats.HeapObjects)) + m.SetGauge([]string{"runtime", "total_gc_pause_ns"}, float32(stats.PauseTotalNs)) + m.SetGauge([]string{"runtime", "total_gc_runs"}, float32(stats.NumGC)) + + // Export info about the last few GC runs + num := stats.NumGC + + // Handle wrap around + if num < m.lastNumGC { + m.lastNumGC = 0 + } + + // Ensure we don't scan more than 256 + if num-m.lastNumGC >= 256 { + m.lastNumGC = num - 255 + } + + for i := m.lastNumGC; i < num; i++ { + pause := stats.PauseNs[i%256] + m.AddSample([]string{"runtime", "gc_pause_ns"}, float32(pause)) + } + m.lastNumGC = num +} + +// Inserts a string value at an index into the slice +func insert(i int, v string, s []string) []string { + s = append(s, "") + copy(s[i+1:], s[i:]) + s[i] = v + return s +} diff --git a/vendor/github.com/armon/go-metrics/sink.go b/vendor/github.com/armon/go-metrics/sink.go new file mode 100644 index 0000000..0b7d6e4 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/sink.go @@ -0,0 +1,115 @@ +package metrics + +import ( + "fmt" + "net/url" +) + +// The MetricSink interface is used to transmit metrics information +// to an external system +type MetricSink interface { + // A Gauge should retain the last value it is set to + SetGauge(key []string, val float32) + SetGaugeWithLabels(key []string, val float32, labels []Label) + + // Should emit a Key/Value pair for each call + EmitKey(key []string, val float32) + + // Counters should accumulate values + IncrCounter(key []string, val float32) + IncrCounterWithLabels(key []string, val float32, labels []Label) + + // Samples are for timing information, where quantiles are used + AddSample(key []string, val float32) + AddSampleWithLabels(key []string, val float32, labels []Label) +} + +// BlackholeSink is used to just blackhole messages +type BlackholeSink struct{} + +func (*BlackholeSink) SetGauge(key []string, val float32) {} +func (*BlackholeSink) SetGaugeWithLabels(key []string, val float32, labels []Label) {} +func (*BlackholeSink) EmitKey(key []string, val float32) {} +func (*BlackholeSink) IncrCounter(key []string, val float32) {} +func (*BlackholeSink) IncrCounterWithLabels(key []string, val float32, labels []Label) {} +func (*BlackholeSink) AddSample(key []string, val float32) {} +func (*BlackholeSink) AddSampleWithLabels(key []string, val float32, labels []Label) {} + +// FanoutSink is used to sink to fanout values to multiple sinks +type FanoutSink []MetricSink + +func (fh FanoutSink) SetGauge(key []string, val float32) { + fh.SetGaugeWithLabels(key, val, nil) +} + +func (fh FanoutSink) SetGaugeWithLabels(key []string, val float32, labels []Label) { + for _, s := range fh { + s.SetGaugeWithLabels(key, val, labels) + } +} + +func (fh FanoutSink) EmitKey(key []string, val float32) { + for _, s := range fh { + s.EmitKey(key, val) + } +} + +func (fh FanoutSink) IncrCounter(key []string, val float32) { + fh.IncrCounterWithLabels(key, val, nil) +} + +func (fh FanoutSink) IncrCounterWithLabels(key []string, val float32, labels []Label) { + for _, s := range fh { + s.IncrCounterWithLabels(key, val, labels) + } +} + +func (fh FanoutSink) AddSample(key []string, val float32) { + fh.AddSampleWithLabels(key, val, nil) +} + +func (fh FanoutSink) AddSampleWithLabels(key []string, val float32, labels []Label) { + for _, s := range fh { + s.AddSampleWithLabels(key, val, labels) + } +} + +// sinkURLFactoryFunc is an generic interface around the *SinkFromURL() function provided +// by each sink type +type sinkURLFactoryFunc func(*url.URL) (MetricSink, error) + +// sinkRegistry supports the generic NewMetricSink function by mapping URL +// schemes to metric sink factory functions +var sinkRegistry = map[string]sinkURLFactoryFunc{ + "statsd": NewStatsdSinkFromURL, + "statsite": NewStatsiteSinkFromURL, + "inmem": NewInmemSinkFromURL, +} + +// NewMetricSinkFromURL allows a generic URL input to configure any of the +// supported sinks. The scheme of the URL identifies the type of the sink, the +// and query parameters are used to set options. +// +// "statsd://" - Initializes a StatsdSink. The host and port are passed through +// as the "addr" of the sink +// +// "statsite://" - Initializes a StatsiteSink. The host and port become the +// "addr" of the sink +// +// "inmem://" - Initializes an InmemSink. The host and port are ignored. The +// "interval" and "duration" query parameters must be specified with valid +// durations, see NewInmemSink for details. +func NewMetricSinkFromURL(urlStr string) (MetricSink, error) { + u, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + + sinkURLFactoryFunc := sinkRegistry[u.Scheme] + if sinkURLFactoryFunc == nil { + return nil, fmt.Errorf( + "cannot create metric sink, unrecognized sink name: %q", u.Scheme) + } + + return sinkURLFactoryFunc(u) +} diff --git a/vendor/github.com/armon/go-metrics/start.go b/vendor/github.com/armon/go-metrics/start.go new file mode 100644 index 0000000..32a28c4 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/start.go @@ -0,0 +1,141 @@ +package metrics + +import ( + "os" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/go-immutable-radix" +) + +// Config is used to configure metrics settings +type Config struct { + ServiceName string // Prefixed with keys to separate services + HostName string // Hostname to use. If not provided and EnableHostname, it will be os.Hostname + EnableHostname bool // Enable prefixing gauge values with hostname + EnableHostnameLabel bool // Enable adding hostname to labels + EnableServiceLabel bool // Enable adding service to labels + EnableRuntimeMetrics bool // Enables profiling of runtime metrics (GC, Goroutines, Memory) + EnableTypePrefix bool // Prefixes key with a type ("counter", "gauge", "timer") + TimerGranularity time.Duration // Granularity of timers. + ProfileInterval time.Duration // Interval to profile runtime metrics + + AllowedPrefixes []string // A list of metric prefixes to allow, with '.' as the separator + BlockedPrefixes []string // A list of metric prefixes to block, with '.' as the separator + AllowedLabels []string // A list of metric labels to allow, with '.' as the separator + BlockedLabels []string // A list of metric labels to block, with '.' as the separator + FilterDefault bool // Whether to allow metrics by default +} + +// Metrics represents an instance of a metrics sink that can +// be used to emit +type Metrics struct { + Config + lastNumGC uint32 + sink MetricSink + filter *iradix.Tree + allowedLabels map[string]bool + blockedLabels map[string]bool + filterLock sync.RWMutex // Lock filters and allowedLabels/blockedLabels access +} + +// Shared global metrics instance +var globalMetrics atomic.Value // *Metrics + +func init() { + // Initialize to a blackhole sink to avoid errors + globalMetrics.Store(&Metrics{sink: &BlackholeSink{}}) +} + +// DefaultConfig provides a sane default configuration +func DefaultConfig(serviceName string) *Config { + c := &Config{ + ServiceName: serviceName, // Use client provided service + HostName: "", + EnableHostname: true, // Enable hostname prefix + EnableRuntimeMetrics: true, // Enable runtime profiling + EnableTypePrefix: false, // Disable type prefix + TimerGranularity: time.Millisecond, // Timers are in milliseconds + ProfileInterval: time.Second, // Poll runtime every second + FilterDefault: true, // Don't filter metrics by default + } + + // Try to get the hostname + name, _ := os.Hostname() + c.HostName = name + return c +} + +// New is used to create a new instance of Metrics +func New(conf *Config, sink MetricSink) (*Metrics, error) { + met := &Metrics{} + met.Config = *conf + met.sink = sink + met.UpdateFilterAndLabels(conf.AllowedPrefixes, conf.BlockedPrefixes, conf.AllowedLabels, conf.BlockedLabels) + + // Start the runtime collector + if conf.EnableRuntimeMetrics { + go met.collectStats() + } + return met, nil +} + +// NewGlobal is the same as New, but it assigns the metrics object to be +// used globally as well as returning it. +func NewGlobal(conf *Config, sink MetricSink) (*Metrics, error) { + metrics, err := New(conf, sink) + if err == nil { + globalMetrics.Store(metrics) + } + return metrics, err +} + +// Proxy all the methods to the globalMetrics instance +func SetGauge(key []string, val float32) { + globalMetrics.Load().(*Metrics).SetGauge(key, val) +} + +func SetGaugeWithLabels(key []string, val float32, labels []Label) { + globalMetrics.Load().(*Metrics).SetGaugeWithLabels(key, val, labels) +} + +func EmitKey(key []string, val float32) { + globalMetrics.Load().(*Metrics).EmitKey(key, val) +} + +func IncrCounter(key []string, val float32) { + globalMetrics.Load().(*Metrics).IncrCounter(key, val) +} + +func IncrCounterWithLabels(key []string, val float32, labels []Label) { + globalMetrics.Load().(*Metrics).IncrCounterWithLabels(key, val, labels) +} + +func AddSample(key []string, val float32) { + globalMetrics.Load().(*Metrics).AddSample(key, val) +} + +func AddSampleWithLabels(key []string, val float32, labels []Label) { + globalMetrics.Load().(*Metrics).AddSampleWithLabels(key, val, labels) +} + +func MeasureSince(key []string, start time.Time) { + globalMetrics.Load().(*Metrics).MeasureSince(key, start) +} + +func MeasureSinceWithLabels(key []string, start time.Time, labels []Label) { + globalMetrics.Load().(*Metrics).MeasureSinceWithLabels(key, start, labels) +} + +func UpdateFilter(allow, block []string) { + globalMetrics.Load().(*Metrics).UpdateFilter(allow, block) +} + +// UpdateFilterAndLabels set allow/block prefixes of metrics while allowedLabels +// and blockedLabels - when not nil - allow filtering of labels in order to +// block/allow globally labels (especially useful when having large number of +// values for a given label). See README.md for more information about usage. +func UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels []string) { + globalMetrics.Load().(*Metrics).UpdateFilterAndLabels(allow, block, allowedLabels, blockedLabels) +} diff --git a/vendor/github.com/armon/go-metrics/statsd.go b/vendor/github.com/armon/go-metrics/statsd.go new file mode 100644 index 0000000..1bfffce --- /dev/null +++ b/vendor/github.com/armon/go-metrics/statsd.go @@ -0,0 +1,184 @@ +package metrics + +import ( + "bytes" + "fmt" + "log" + "net" + "net/url" + "strings" + "time" +) + +const ( + // statsdMaxLen is the maximum size of a packet + // to send to statsd + statsdMaxLen = 1400 +) + +// StatsdSink provides a MetricSink that can be used +// with a statsite or statsd metrics server. It uses +// only UDP packets, while StatsiteSink uses TCP. +type StatsdSink struct { + addr string + metricQueue chan string +} + +// NewStatsdSinkFromURL creates an StatsdSink from a URL. It is used +// (and tested) from NewMetricSinkFromURL. +func NewStatsdSinkFromURL(u *url.URL) (MetricSink, error) { + return NewStatsdSink(u.Host) +} + +// NewStatsdSink is used to create a new StatsdSink +func NewStatsdSink(addr string) (*StatsdSink, error) { + s := &StatsdSink{ + addr: addr, + metricQueue: make(chan string, 4096), + } + go s.flushMetrics() + return s, nil +} + +// Close is used to stop flushing to statsd +func (s *StatsdSink) Shutdown() { + close(s.metricQueue) +} + +func (s *StatsdSink) SetGauge(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsdSink) SetGaugeWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsdSink) EmitKey(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val)) +} + +func (s *StatsdSink) IncrCounter(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsdSink) IncrCounterWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsdSink) AddSample(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +func (s *StatsdSink) AddSampleWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +// Flattens the key for formatting, removes spaces +func (s *StatsdSink) flattenKey(parts []string) string { + joined := strings.Join(parts, ".") + return strings.Map(func(r rune) rune { + switch r { + case ':': + fallthrough + case ' ': + return '_' + default: + return r + } + }, joined) +} + +// Flattens the key along with labels for formatting, removes spaces +func (s *StatsdSink) flattenKeyLabels(parts []string, labels []Label) string { + for _, label := range labels { + parts = append(parts, label.Value) + } + return s.flattenKey(parts) +} + +// Does a non-blocking push to the metrics queue +func (s *StatsdSink) pushMetric(m string) { + select { + case s.metricQueue <- m: + default: + } +} + +// Flushes metrics +func (s *StatsdSink) flushMetrics() { + var sock net.Conn + var err error + var wait <-chan time.Time + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + +CONNECT: + // Create a buffer + buf := bytes.NewBuffer(nil) + + // Attempt to connect + sock, err = net.Dial("udp", s.addr) + if err != nil { + log.Printf("[ERR] Error connecting to statsd! Err: %s", err) + goto WAIT + } + + for { + select { + case metric, ok := <-s.metricQueue: + // Get a metric from the queue + if !ok { + goto QUIT + } + + // Check if this would overflow the packet size + if len(metric)+buf.Len() > statsdMaxLen { + _, err := sock.Write(buf.Bytes()) + buf.Reset() + if err != nil { + log.Printf("[ERR] Error writing to statsd! Err: %s", err) + goto WAIT + } + } + + // Append to the buffer + buf.WriteString(metric) + + case <-ticker.C: + if buf.Len() == 0 { + continue + } + + _, err := sock.Write(buf.Bytes()) + buf.Reset() + if err != nil { + log.Printf("[ERR] Error flushing to statsd! Err: %s", err) + goto WAIT + } + } + } + +WAIT: + // Wait for a while + wait = time.After(time.Duration(5) * time.Second) + for { + select { + // Dequeue the messages to avoid backlog + case _, ok := <-s.metricQueue: + if !ok { + goto QUIT + } + case <-wait: + goto CONNECT + } + } +QUIT: + s.metricQueue = nil +} diff --git a/vendor/github.com/armon/go-metrics/statsite.go b/vendor/github.com/armon/go-metrics/statsite.go new file mode 100644 index 0000000..6c0d284 --- /dev/null +++ b/vendor/github.com/armon/go-metrics/statsite.go @@ -0,0 +1,172 @@ +package metrics + +import ( + "bufio" + "fmt" + "log" + "net" + "net/url" + "strings" + "time" +) + +const ( + // We force flush the statsite metrics after this period of + // inactivity. Prevents stats from getting stuck in a buffer + // forever. + flushInterval = 100 * time.Millisecond +) + +// NewStatsiteSinkFromURL creates an StatsiteSink from a URL. It is used +// (and tested) from NewMetricSinkFromURL. +func NewStatsiteSinkFromURL(u *url.URL) (MetricSink, error) { + return NewStatsiteSink(u.Host) +} + +// StatsiteSink provides a MetricSink that can be used with a +// statsite metrics server +type StatsiteSink struct { + addr string + metricQueue chan string +} + +// NewStatsiteSink is used to create a new StatsiteSink +func NewStatsiteSink(addr string) (*StatsiteSink, error) { + s := &StatsiteSink{ + addr: addr, + metricQueue: make(chan string, 4096), + } + go s.flushMetrics() + return s, nil +} + +// Close is used to stop flushing to statsite +func (s *StatsiteSink) Shutdown() { + close(s.metricQueue) +} + +func (s *StatsiteSink) SetGauge(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsiteSink) SetGaugeWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|g\n", flatKey, val)) +} + +func (s *StatsiteSink) EmitKey(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|kv\n", flatKey, val)) +} + +func (s *StatsiteSink) IncrCounter(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsiteSink) IncrCounterWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|c\n", flatKey, val)) +} + +func (s *StatsiteSink) AddSample(key []string, val float32) { + flatKey := s.flattenKey(key) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +func (s *StatsiteSink) AddSampleWithLabels(key []string, val float32, labels []Label) { + flatKey := s.flattenKeyLabels(key, labels) + s.pushMetric(fmt.Sprintf("%s:%f|ms\n", flatKey, val)) +} + +// Flattens the key for formatting, removes spaces +func (s *StatsiteSink) flattenKey(parts []string) string { + joined := strings.Join(parts, ".") + return strings.Map(func(r rune) rune { + switch r { + case ':': + fallthrough + case ' ': + return '_' + default: + return r + } + }, joined) +} + +// Flattens the key along with labels for formatting, removes spaces +func (s *StatsiteSink) flattenKeyLabels(parts []string, labels []Label) string { + for _, label := range labels { + parts = append(parts, label.Value) + } + return s.flattenKey(parts) +} + +// Does a non-blocking push to the metrics queue +func (s *StatsiteSink) pushMetric(m string) { + select { + case s.metricQueue <- m: + default: + } +} + +// Flushes metrics +func (s *StatsiteSink) flushMetrics() { + var sock net.Conn + var err error + var wait <-chan time.Time + var buffered *bufio.Writer + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + +CONNECT: + // Attempt to connect + sock, err = net.Dial("tcp", s.addr) + if err != nil { + log.Printf("[ERR] Error connecting to statsite! Err: %s", err) + goto WAIT + } + + // Create a buffered writer + buffered = bufio.NewWriter(sock) + + for { + select { + case metric, ok := <-s.metricQueue: + // Get a metric from the queue + if !ok { + goto QUIT + } + + // Try to send to statsite + _, err := buffered.Write([]byte(metric)) + if err != nil { + log.Printf("[ERR] Error writing to statsite! Err: %s", err) + goto WAIT + } + case <-ticker.C: + if err := buffered.Flush(); err != nil { + log.Printf("[ERR] Error flushing to statsite! Err: %s", err) + goto WAIT + } + } + } + +WAIT: + // Wait for a while + wait = time.After(time.Duration(5) * time.Second) + for { + select { + // Dequeue the messages to avoid backlog + case _, ok := <-s.metricQueue: + if !ok { + goto QUIT + } + case <-wait: + goto CONNECT + } + } +QUIT: + s.metricQueue = nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ee9fb46..c2e0aa3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,3 +1,5 @@ +# github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da +github.com/armon/go-metrics # github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 github.com/armon/go-radix # github.com/go-errors/errors v1.0.1 @@ -36,8 +38,8 @@ github.com/hashicorp/go-uuid # github.com/hashicorp/go-version v1.1.0 github.com/hashicorp/go-version # github.com/hashicorp/golang-lru v0.5.1 -github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru +github.com/hashicorp/golang-lru # github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hcl github.com/hashicorp/hcl/hcl/ast @@ -54,6 +56,8 @@ github.com/hashicorp/vault/api github.com/hashicorp/vault/sdk/plugin github.com/hashicorp/vault/sdk/framework github.com/hashicorp/vault/sdk/helper/ldaputil +github.com/hashicorp/vault/sdk/helper/locksutil +github.com/hashicorp/vault/sdk/helper/strutil github.com/hashicorp/vault/sdk/logical github.com/hashicorp/vault/sdk/helper/consts github.com/hashicorp/vault/sdk/helper/hclutil @@ -66,17 +70,15 @@ github.com/hashicorp/vault/sdk/plugin/pb github.com/hashicorp/vault/sdk/helper/errutil github.com/hashicorp/vault/sdk/helper/logging github.com/hashicorp/vault/sdk/helper/salt -github.com/hashicorp/vault/sdk/helper/strutil github.com/hashicorp/vault/sdk/version github.com/hashicorp/vault/sdk/helper/tlsutil +github.com/hashicorp/vault/sdk/helper/cryptoutil github.com/hashicorp/vault/sdk/physical github.com/hashicorp/vault/sdk/physical/inmem github.com/hashicorp/vault/sdk/helper/compressutil github.com/hashicorp/vault/sdk/helper/certutil github.com/hashicorp/vault/sdk/helper/mlock -github.com/hashicorp/vault/sdk/helper/locksutil github.com/hashicorp/vault/sdk/helper/pathmanager -github.com/hashicorp/vault/sdk/helper/cryptoutil # github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d github.com/hashicorp/yamux # github.com/mitchellh/go-homedir v1.1.0 @@ -95,12 +97,12 @@ github.com/pierrec/lz4/internal/xxh32 # github.com/ryanuber/go-glob v1.0.0 github.com/ryanuber/go-glob # golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 +golang.org/x/crypto/blake2b golang.org/x/crypto/ed25519 golang.org/x/crypto/pbkdf2 golang.org/x/crypto/cryptobyte golang.org/x/crypto/cryptobyte/asn1 golang.org/x/crypto/ed25519/internal/edwards25519 -golang.org/x/crypto/blake2b # golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/net/http2 golang.org/x/net/http/httpguts