Skip to content

Commit

Permalink
Port: Add metrics to report mount table sizes for auth and logical [V…
Browse files Browse the repository at this point in the history 
…ault 671] (#10201)

* first commit

* update

* removed some ent features from backport

* final refactor

* backport patch

Co-authored-by: Hridoy Roy <[email protected]>
Co-authored-by: Hridoy Roy <[email protected]>
  • Loading branch information
@HridoyRoy
3 people authored Oct 27, 2020
1 parent 20c95cd commit aac17a1
Show file tree
Hide file tree
Showing 9 changed files with 481 additions and 61 deletions.
43 changes: 42 additions & 1 deletion helper/metricsutil/metricsutil.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"net/http"
"strings"
"sync"

"github.com/armon/go-metrics"
"github.com/hashicorp/vault/sdk/logical"
Expand All @@ -26,13 +27,34 @@ const (
PrometheusMetricFormat = "prometheus"
)

// PhysicalTableSizeName is a set of gauge metric keys for physical mount table sizes
var PhysicalTableSizeName []string = []string{"core", "mount_table", "size"}

// LogicalTableSizeName is a set of gauge metric keys for logical mount table sizes
var LogicalTableSizeName []string = []string{"core", "mount_table", "num_entries"}

type MetricsHelper struct {
inMemSink *metrics.InmemSink
PrometheusEnabled bool
LoopMetrics GaugeMetrics
}

type GaugeMetrics struct {
// Metrics is a map from keys concatenated by "." to the metric.
// It is a map because although we do not care about distinguishing
// these loop metrics during emission, we must distinguish them
// when we update a metric.
Metrics sync.Map
}

type GaugeMetric struct {
Value float32
Labels []Label
Key []string
}

func NewMetricsHelper(inMem *metrics.InmemSink, enablePrometheus bool) *MetricsHelper {
return &MetricsHelper{inMem, enablePrometheus}
return &MetricsHelper{inMem, enablePrometheus, GaugeMetrics{Metrics: sync.Map{}}}
}

func FormatFromRequest(req *logical.Request) string {
Expand All @@ -53,6 +75,25 @@ func FormatFromRequest(req *logical.Request) string {
return ""
}

func (m *MetricsHelper) AddGaugeLoopMetric(key []string, val float32, labels []Label) {
mapKey := m.CreateMetricsCacheKeyName(key, val, labels)
m.LoopMetrics.Metrics.Store(mapKey,
GaugeMetric{
Key: key,
Value: val,
Labels: labels})
}

func (m *MetricsHelper) CreateMetricsCacheKeyName(key []string, val float32, labels []Label) string {
var keyJoin string = strings.Join(key, ".")
var labelJoinStr = ""
for _, label := range labels {
labelJoinStr = labelJoinStr + label.Name + "|" + label.Value + "||"
}
keyJoin = keyJoin + "." + labelJoinStr
return keyJoin
}

func (m *MetricsHelper) ResponseForFormat(format string) *logical.Response {
switch format {
case PrometheusMetricFormat:
Expand Down
32 changes: 23 additions & 9 deletions vault/auth.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -478,8 +478,10 @@ func (c *Core) loadCredentials(ctx context.Context) error {
if c.auth == nil {
c.auth = c.defaultAuthTable()
needPersist = true
} else {
// only record tableMetrics if we have loaded something from storge
c.tableMetrics(len(c.auth.Entries), false, true, raw.Value)
}

if rawLocal != nil {
localAuthTable, err := c.decodeMountTable(ctx, rawLocal.Value)
if err != nil {
Expand All @@ -488,6 +490,7 @@ func (c *Core) loadCredentials(ctx context.Context) error {
}
if localAuthTable != nil && len(localAuthTable.Entries) > 0 {
c.auth.Entries = append(c.auth.Entries, localAuthTable.Entries...)
c.tableMetrics(len(localAuthTable.Entries), true, true, rawLocal.Value)
}
}

Expand Down Expand Up @@ -579,12 +582,12 @@ func (c *Core) persistAuth(ctx context.Context, table *MountTable, local *bool)
}
}

writeTable := func(mt *MountTable, path string) error {
writeTable := func(mt *MountTable, path string) ([]byte, error) {
// Encode the mount table into JSON and compress it (lzw).
compressedBytes, err := jsonutil.EncodeJSONAndCompress(mt, nil)
if err != nil {
c.logger.Error("failed to encode or compress auth mount table", "error", err)
return err
return nil, err
}

// Create an entry
Expand All @@ -596,29 +599,40 @@ func (c *Core) persistAuth(ctx context.Context, table *MountTable, local *bool)
// Write to the physical backend
if err := c.barrier.Put(ctx, entry); err != nil {
c.logger.Error("failed to persist auth mount table", "error", err)
return err
return nil, err
}
return nil
return compressedBytes, nil
}

var err error
var compressedBytes []byte
switch {
case local == nil:
// Write non-local mounts
err := writeTable(nonLocalAuth, coreAuthConfigPath)
compressedBytes, err := writeTable(nonLocalAuth, coreAuthConfigPath)
if err != nil {
return err
}
c.tableMetrics(len(nonLocalAuth.Entries), false, true, compressedBytes)

// Write local mounts
err = writeTable(localAuth, coreLocalAuthConfigPath)
compressedBytes, err = writeTable(localAuth, coreLocalAuthConfigPath)
if err != nil {
return err
}
c.tableMetrics(len(localAuth.Entries), true, true, compressedBytes)
case *local:
err = writeTable(localAuth, coreLocalAuthConfigPath)
compressedBytes, err = writeTable(localAuth, coreLocalAuthConfigPath)
if err != nil {
return err
}
c.tableMetrics(len(localAuth.Entries), true, true, compressedBytes)
default:
err = writeTable(nonLocalAuth, coreAuthConfigPath)
compressedBytes, err = writeTable(nonLocalAuth, coreAuthConfigPath)
if err != nil {
return err
}
c.tableMetrics(len(nonLocalAuth.Entries), false, true, compressedBytes)
}

return err
Expand Down
118 changes: 100 additions & 18 deletions vault/auth_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@ import (
"reflect"
"strings"
"testing"
"time"

metrics "github.com/armon/go-metrics"
"github.com/hashicorp/vault/helper/metricsutil"
"github.com/hashicorp/vault/helper/namespace"
"github.com/hashicorp/vault/sdk/helper/jsonutil"
"github.com/hashicorp/vault/sdk/logical"
)

func TestAuth_ReadOnlyViewDuringMount(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(ctx context.Context, config *logical.BackendConfig) (logical.Backend, error) {
err := config.StorageView.Put(ctx, &logical.StorageEntry{
Key: "bar",
Expand All @@ -37,14 +40,85 @@ func TestAuth_ReadOnlyViewDuringMount(t *testing.T) {
}
}

func TestAuthMountMetrics(t *testing.T) {
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(ctx context.Context, config *logical.BackendConfig) (logical.Backend, error) {
return &NoopBackend{
BackendType: logical.TypeCredential,
}, nil
}
mountKeyName := "core.mount_table.num_entries.type|auth||local|false||"
mountMetrics := &c.metricsHelper.LoopMetrics.Metrics
loadMetric, ok := mountMetrics.Load(mountKeyName)
var numEntriesMetric metricsutil.GaugeMetric = loadMetric.(metricsutil.GaugeMetric)

// 1 default nonlocal auth backend
if !ok || numEntriesMetric.Value != 1 {
t.Fatalf("Auth values should be: %+v", numEntriesMetric)
}

me := &MountEntry{
Table: credentialTableType,
Path: "foo",
Type: "noop",
}
err := c.enableCredential(namespace.RootContext(nil), me)
if err != nil {
t.Fatalf("err: %v", err)
}
mountMetrics = &c.metricsHelper.LoopMetrics.Metrics
loadMetric, ok = mountMetrics.Load(mountKeyName)
numEntriesMetric = loadMetric.(metricsutil.GaugeMetric)
if !ok || numEntriesMetric.Value != 2 {
t.Fatalf("mount metrics for num entries do not match true values")
}
if len(numEntriesMetric.Key) != 3 ||
numEntriesMetric.Key[0] != "core" ||
numEntriesMetric.Key[1] != "mount_table" ||
numEntriesMetric.Key[2] != "num_entries" {
t.Fatalf("mount metrics for num entries have wrong key")
}
if len(numEntriesMetric.Labels) != 2 ||
numEntriesMetric.Labels[0].Name != "type" ||
numEntriesMetric.Labels[0].Value != "auth" ||
numEntriesMetric.Labels[1].Name != "local" ||
numEntriesMetric.Labels[1].Value != "false" {
t.Fatalf("mount metrics for num entries have wrong labels")
}
mountSizeKeyName := "core.mount_table.size.type|auth||local|false||"
loadMetric, ok = mountMetrics.Load(mountSizeKeyName)
sizeMetric := loadMetric.(metricsutil.GaugeMetric)

if !ok {
t.Fatalf("mount metrics for size do not match exist")
}
if len(sizeMetric.Key) != 3 ||
sizeMetric.Key[0] != "core" ||
sizeMetric.Key[1] != "mount_table" ||
sizeMetric.Key[2] != "size" {
t.Fatalf("mount metrics for size have wrong key")
}
if len(sizeMetric.Labels) != 2 ||
sizeMetric.Labels[0].Name != "type" ||
sizeMetric.Labels[0].Value != "auth" ||
sizeMetric.Labels[1].Name != "local" ||
sizeMetric.Labels[1].Value != "false" {
t.Fatalf("mount metrics for size have wrong labels")
}
}

func TestCore_DefaultAuthTable(t *testing.T) {
c, keys, _ := TestCoreUnsealed(t)
c, keys, _, _ := TestCoreUnsealedWithMetrics(t)
verifyDefaultAuthTable(t, c.auth)

// Start a second core with same physical
inmemSink := metrics.NewInmemSink(1000000*time.Hour, 2000000*time.Hour)
conf := &CoreConfig{
Physical: c.physical,
DisableMlock: true,
Physical: c.physical,
DisableMlock: true,
BuiltinRegistry: NewMockBuiltinRegistry(),
MetricSink: metricsutil.NewClusterMetricSink("test-cluster", inmemSink),
MetricsHelper: metricsutil.NewMetricsHelper(inmemSink, false),
}
c2, err := NewCore(conf)
if err != nil {
Expand All @@ -67,7 +141,7 @@ func TestCore_DefaultAuthTable(t *testing.T) {
}

func TestCore_EnableCredential(t *testing.T) {
c, keys, _ := TestCoreUnsealed(t)
c, keys, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return &NoopBackend{
BackendType: logical.TypeCredential,
Expand All @@ -89,9 +163,13 @@ func TestCore_EnableCredential(t *testing.T) {
t.Fatalf("missing mount, match: %q", match)
}

inmemSink := metrics.NewInmemSink(1000000*time.Hour, 2000000*time.Hour)
conf := &CoreConfig{
Physical: c.physical,
DisableMlock: true,
Physical: c.physical,
DisableMlock: true,
BuiltinRegistry: NewMockBuiltinRegistry(),
MetricSink: metricsutil.NewClusterMetricSink("test-cluster", inmemSink),
MetricsHelper: metricsutil.NewMetricsHelper(inmemSink, false),
}
c2, err := NewCore(conf)
if err != nil {
Expand Down Expand Up @@ -122,7 +200,7 @@ func TestCore_EnableCredential(t *testing.T) {
// entries, and that upon reading the entries from both are recombined
// correctly
func TestCore_EnableCredential_Local(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return &NoopBackend{
BackendType: logical.TypeCredential,
Expand Down Expand Up @@ -211,7 +289,7 @@ func TestCore_EnableCredential_Local(t *testing.T) {
}

func TestCore_EnableCredential_twice_409(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return &NoopBackend{
BackendType: logical.TypeCredential,
Expand Down Expand Up @@ -241,7 +319,7 @@ func TestCore_EnableCredential_twice_409(t *testing.T) {
}

func TestCore_EnableCredential_Token(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
me := &MountEntry{
Table: credentialTableType,
Path: "foo",
Expand All @@ -254,7 +332,7 @@ func TestCore_EnableCredential_Token(t *testing.T) {
}

func TestCore_DisableCredential(t *testing.T) {
c, keys, _ := TestCoreUnsealed(t)
c, keys, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return &NoopBackend{
BackendType: logical.TypeCredential,
Expand Down Expand Up @@ -286,9 +364,13 @@ func TestCore_DisableCredential(t *testing.T) {
t.Fatalf("backend present")
}

inmemSink := metrics.NewInmemSink(1000000*time.Hour, 2000000*time.Hour)
conf := &CoreConfig{
Physical: c.physical,
DisableMlock: true,
Physical: c.physical,
DisableMlock: true,
BuiltinRegistry: NewMockBuiltinRegistry(),
MetricSink: metricsutil.NewClusterMetricSink("test-cluster", inmemSink),
MetricsHelper: metricsutil.NewMetricsHelper(inmemSink, false),
}
c2, err := NewCore(conf)
if err != nil {
Expand All @@ -311,7 +393,7 @@ func TestCore_DisableCredential(t *testing.T) {
}

func TestCore_DisableCredential_Protected(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
err := c.disableCredential(namespace.RootContext(nil), "token")
if err.Error() != "token credential backend cannot be disabled" {
t.Fatalf("err: %v", err)
Expand All @@ -323,7 +405,7 @@ func TestCore_DisableCredential_Cleanup(t *testing.T) {
Login: []string{"login"},
BackendType: logical.TypeCredential,
}
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["noop"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return noop, nil
}
Expand Down Expand Up @@ -394,7 +476,7 @@ func TestCore_DisableCredential_Cleanup(t *testing.T) {
}

func TestDefaultAuthTable(t *testing.T) {
c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
table := c.defaultAuthTable()
verifyDefaultAuthTable(t, table)
}
Expand Down Expand Up @@ -432,7 +514,7 @@ func TestCore_CredentialInitialize(t *testing.T) {
BackendType: logical.TypeCredential,
}, false}

c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return backend, nil
}
Expand All @@ -457,7 +539,7 @@ func TestCore_CredentialInitialize(t *testing.T) {
BackendType: logical.TypeCredential,
}, false}

c, _, _ := TestCoreUnsealed(t)
c, _, _, _ := TestCoreUnsealedWithMetrics(t)
c.credentialBackends["initable"] = func(context.Context, *logical.BackendConfig) (logical.Backend, error) {
return backend, nil
}
Expand Down
Loading

0 comments on commit aac17a1

Please sign in to comment.