From ec787da92328eb32ec0e0569c9e032b9b94e6e01 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 18 Dec 2024 15:49:55 +0100 Subject: [PATCH 1/4] Add types for Go runtime metrics * Adds realtime metrics type * Adds profile type * Remove legacy profile calls --- metrics.go | 70 +++- metrics_gen.go | 758 ++++++++++++++++++++++++++++++++++++++++++ metrics_gen_test.go | 226 +++++++++++++ profiling-commands.go | 64 +--- 4 files changed, 1054 insertions(+), 64 deletions(-) diff --git a/metrics.go b/metrics.go index d651859..c913bc0 100644 --- a/metrics.go +++ b/metrics.go @@ -27,6 +27,7 @@ import ( "io" "net/http" "net/url" + "runtime/metrics" "sort" "strconv" "strings" @@ -39,7 +40,7 @@ import ( //msgp:clearomitted //msgp:tag json -//go:generate msgp +//go:generate msgp -unexported // MetricType is a bitfield representation of different metric types. type MetricType uint32 @@ -57,6 +58,7 @@ const ( MetricsMem MetricsCPU MetricsRPC + MetricsRuntime // MetricsAll must be last. // Enables all metrics. @@ -162,6 +164,7 @@ type Metrics struct { Mem *MemMetrics `json:"mem,omitempty"` CPU *CPUMetrics `json:"cpu,omitempty"` RPC *RPCMetrics `json:"rpc,omitempty"` + Go *RuntimeMetrics `json:"go,omitempty"` } // Merge other into r. @@ -201,6 +204,10 @@ func (r *Metrics) Merge(other *Metrics) { r.RPC = &RPCMetrics{} } r.RPC.Merge(other.RPC) + if r.Go == nil && other.Go != nil { + r.Go = &RuntimeMetrics{} + } + r.Go.Merge(other.Go) } // Merge will merge other into r. @@ -756,3 +763,64 @@ func (m *RPCMetrics) Merge(other *RPCMetrics) { m.ByCaller[k] = existing } } + +//msgp:replace metrics.Float64Histogram with:localF64H + +// local copy of localF64H, can be casted to/from metrics.Float64Histogram +type localF64H struct { + Counts []uint64 `json:"counts,omitempty"` + Buckets []float64 `json:"buckets,omitempty"` +} + +// RuntimeMetrics contains metrics for the go runtime. +// See more at https://pkg.go.dev/runtime/metric +type RuntimeMetrics struct { + // UintMetrics contains KindUint64 values + UintMetrics map[string]uint64 `json:"uintMetrics,omitempty"` + + // FloatMetrics contains KindFloat64 values + FloatMetrics map[string]float64 `json:"floatMetrics,omitempty"` + // HistMetrics contains KindFloat64Histogram values + + HistMetrics map[string]metrics.Float64Histogram `json:"histMetrics,omitempty"` + + // N tracks the number of merged entries. + N int `json:"n"` +} + +// Merge other into 'm'. +func (m *RuntimeMetrics) Merge(other *RuntimeMetrics) { + if m == nil || other == nil { + return + } + if m.UintMetrics == nil { + m.UintMetrics = make(map[string]uint64) + } + if m.FloatMetrics == nil { + m.UintMetrics = make(map[string]uint64) + } + if m.FloatMetrics == nil { + m.UintMetrics = make(map[string]uint64) + } + for k, v := range other.UintMetrics { + m.UintMetrics[k] += v + } + for k, v := range other.FloatMetrics { + m.FloatMetrics[k] += v + } + for k, v := range other.HistMetrics { + existing := m.HistMetrics[k] + if len(existing.Buckets) == 0 { + m.HistMetrics[k] = v + continue + } + // TODO: Technically, I guess we may have differing buckets, + // but they should be the same for the runtime. + if len(existing.Buckets) == len(v.Buckets) { + for i, count := range v.Counts { + existing.Counts[i] += count + } + } + } + m.N += other.N +} diff --git a/metrics_gen.go b/metrics_gen.go index 7a1754f..51c04b3 100644 --- a/metrics_gen.go +++ b/metrics_gen.go @@ -3,6 +3,7 @@ package madmin // Code generated by github.com/tinylib/msgp DO NOT EDIT. import ( + "runtime/metrics" "time" "github.com/shirou/gopsutil/v3/cpu" @@ -6846,6 +6847,492 @@ func (z *ReplicateInfo) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *RuntimeMetrics) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "uintMetrics": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + if z.UintMetrics == nil { + z.UintMetrics = make(map[string]uint64, zb0002) + } else if len(z.UintMetrics) > 0 { + for key := range z.UintMetrics { + delete(z.UintMetrics, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 uint64 + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + za0002, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "UintMetrics", za0001) + return + } + z.UintMetrics[za0001] = za0002 + } + zb0001Mask |= 0x1 + case "floatMetrics": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + if z.FloatMetrics == nil { + z.FloatMetrics = make(map[string]float64, zb0003) + } else if len(z.FloatMetrics) > 0 { + for key := range z.FloatMetrics { + delete(z.FloatMetrics, key) + } + } + for zb0003 > 0 { + zb0003-- + var za0003 string + var za0004 float64 + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + za0004, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "FloatMetrics", za0003) + return + } + z.FloatMetrics[za0003] = za0004 + } + zb0001Mask |= 0x2 + case "histMetrics": + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + if z.HistMetrics == nil { + z.HistMetrics = make(map[string]metrics.Float64Histogram, zb0004) + } else if len(z.HistMetrics) > 0 { + for key := range z.HistMetrics { + delete(z.HistMetrics, key) + } + } + for zb0004 > 0 { + zb0004-- + var za0005 string + var za0006 metrics.Float64Histogram + za0005, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + err = (*localF64H)(&za0006).DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "HistMetrics", za0005) + return + } + z.HistMetrics[za0005] = za0006 + } + zb0001Mask |= 0x4 + case "n": + z.N, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.UintMetrics = nil + } + if (zb0001Mask & 0x2) == 0 { + z.FloatMetrics = nil + } + if (zb0001Mask & 0x4) == 0 { + z.HistMetrics = nil + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RuntimeMetrics) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(4) + var zb0001Mask uint8 /* 4 bits */ + _ = zb0001Mask + if z.UintMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.FloatMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.HistMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not omitted + // write "uintMetrics" + err = en.Append(0xab, 0x75, 0x69, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.UintMetrics))) + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + for za0001, za0002 := range z.UintMetrics { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + err = en.WriteUint64(za0002) + if err != nil { + err = msgp.WrapError(err, "UintMetrics", za0001) + return + } + } + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "floatMetrics" + err = en.Append(0xac, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.FloatMetrics))) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + for za0003, za0004 := range z.FloatMetrics { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + err = en.WriteFloat64(za0004) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics", za0003) + return + } + } + } + if (zb0001Mask & 0x4) == 0 { // if not omitted + // write "histMetrics" + err = en.Append(0xab, 0x68, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.HistMetrics))) + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + for za0005, za0006 := range z.HistMetrics { + err = en.WriteString(za0005) + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + err = (*localF64H)(&za0006).EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "HistMetrics", za0005) + return + } + } + } + // write "n" + err = en.Append(0xa1, 0x6e) + if err != nil { + return + } + err = en.WriteInt(z.N) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RuntimeMetrics) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(4) + var zb0001Mask uint8 /* 4 bits */ + _ = zb0001Mask + if z.UintMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.FloatMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.HistMetrics == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not omitted + // string "uintMetrics" + o = append(o, 0xab, 0x75, 0x69, 0x6e, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.UintMetrics))) + for za0001, za0002 := range z.UintMetrics { + o = msgp.AppendString(o, za0001) + o = msgp.AppendUint64(o, za0002) + } + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "floatMetrics" + o = append(o, 0xac, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.FloatMetrics))) + for za0003, za0004 := range z.FloatMetrics { + o = msgp.AppendString(o, za0003) + o = msgp.AppendFloat64(o, za0004) + } + } + if (zb0001Mask & 0x4) == 0 { // if not omitted + // string "histMetrics" + o = append(o, 0xab, 0x68, 0x69, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.HistMetrics))) + for za0005, za0006 := range z.HistMetrics { + o = msgp.AppendString(o, za0005) + o, err = (*localF64H)(&za0006).MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "HistMetrics", za0005) + return + } + } + } + // string "n" + o = append(o, 0xa1, 0x6e) + o = msgp.AppendInt(o, z.N) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RuntimeMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "uintMetrics": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + if z.UintMetrics == nil { + z.UintMetrics = make(map[string]uint64, zb0002) + } else if len(z.UintMetrics) > 0 { + for key := range z.UintMetrics { + delete(z.UintMetrics, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 uint64 + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UintMetrics") + return + } + za0002, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UintMetrics", za0001) + return + } + z.UintMetrics[za0001] = za0002 + } + zb0001Mask |= 0x1 + case "floatMetrics": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + if z.FloatMetrics == nil { + z.FloatMetrics = make(map[string]float64, zb0003) + } else if len(z.FloatMetrics) > 0 { + for key := range z.FloatMetrics { + delete(z.FloatMetrics, key) + } + } + for zb0003 > 0 { + var za0003 string + var za0004 float64 + zb0003-- + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics") + return + } + za0004, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FloatMetrics", za0003) + return + } + z.FloatMetrics[za0003] = za0004 + } + zb0001Mask |= 0x2 + case "histMetrics": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + if z.HistMetrics == nil { + z.HistMetrics = make(map[string]metrics.Float64Histogram, zb0004) + } else if len(z.HistMetrics) > 0 { + for key := range z.HistMetrics { + delete(z.HistMetrics, key) + } + } + for zb0004 > 0 { + var za0005 string + var za0006 metrics.Float64Histogram + zb0004-- + za0005, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HistMetrics") + return + } + bts, err = (*localF64H)(&za0006).UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "HistMetrics", za0005) + return + } + z.HistMetrics[za0005] = za0006 + } + zb0001Mask |= 0x4 + case "n": + z.N, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.UintMetrics = nil + } + if (zb0001Mask & 0x2) == 0 { + z.FloatMetrics = nil + } + if (zb0001Mask & 0x4) == 0 { + z.HistMetrics = nil + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RuntimeMetrics) Msgsize() (s int) { + s = 1 + 12 + msgp.MapHeaderSize + if z.UintMetrics != nil { + for za0001, za0002 := range z.UintMetrics { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.Uint64Size + } + } + s += 13 + msgp.MapHeaderSize + if z.FloatMetrics != nil { + for za0003, za0004 := range z.FloatMetrics { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + msgp.Float64Size + } + } + s += 12 + msgp.MapHeaderSize + if z.HistMetrics != nil { + for za0005, za0006 := range z.HistMetrics { + _ = za0006 + s += msgp.StringPrefixSize + len(za0005) + (*localF64H)(&za0006).Msgsize() + } + } + s += 2 + msgp.IntSize + return +} + // DecodeMsg implements msgp.Decodable func (z *ScannerMetrics) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -8475,3 +8962,274 @@ func (z *SiteResyncMetrics) Msgsize() (s int) { s += 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) return } + +// DecodeMsg implements msgp.Decodable +func (z *localF64H) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "counts": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Counts") + return + } + if cap(z.Counts) >= int(zb0002) { + z.Counts = (z.Counts)[:zb0002] + } else { + z.Counts = make([]uint64, zb0002) + } + for za0001 := range z.Counts { + z.Counts[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Counts", za0001) + return + } + } + zb0001Mask |= 0x1 + case "buckets": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + if cap(z.Buckets) >= int(zb0003) { + z.Buckets = (z.Buckets)[:zb0003] + } else { + z.Buckets = make([]float64, zb0003) + } + for za0002 := range z.Buckets { + z.Buckets[za0002], err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Buckets", za0002) + return + } + } + zb0001Mask |= 0x2 + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x3 { + if (zb0001Mask & 0x1) == 0 { + z.Counts = nil + } + if (zb0001Mask & 0x2) == 0 { + z.Buckets = nil + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *localF64H) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + if z.Counts == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.Buckets == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not omitted + // write "counts" + err = en.Append(0xa6, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Counts))) + if err != nil { + err = msgp.WrapError(err, "Counts") + return + } + for za0001 := range z.Counts { + err = en.WriteUint64(z.Counts[za0001]) + if err != nil { + err = msgp.WrapError(err, "Counts", za0001) + return + } + } + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "buckets" + err = en.Append(0xa7, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Buckets))) + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + for za0002 := range z.Buckets { + err = en.WriteFloat64(z.Buckets[za0002]) + if err != nil { + err = msgp.WrapError(err, "Buckets", za0002) + return + } + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *localF64H) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(2) + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + if z.Counts == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + if z.Buckets == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + if (zb0001Mask & 0x1) == 0 { // if not omitted + // string "counts" + o = append(o, 0xa6, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Counts))) + for za0001 := range z.Counts { + o = msgp.AppendUint64(o, z.Counts[za0001]) + } + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "buckets" + o = append(o, 0xa7, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Buckets))) + for za0002 := range z.Buckets { + o = msgp.AppendFloat64(o, z.Buckets[za0002]) + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *localF64H) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "counts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Counts") + return + } + if cap(z.Counts) >= int(zb0002) { + z.Counts = (z.Counts)[:zb0002] + } else { + z.Counts = make([]uint64, zb0002) + } + for za0001 := range z.Counts { + z.Counts[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Counts", za0001) + return + } + } + zb0001Mask |= 0x1 + case "buckets": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + if cap(z.Buckets) >= int(zb0003) { + z.Buckets = (z.Buckets)[:zb0003] + } else { + z.Buckets = make([]float64, zb0003) + } + for za0002 := range z.Buckets { + z.Buckets[za0002], bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Buckets", za0002) + return + } + } + zb0001Mask |= 0x2 + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x3 { + if (zb0001Mask & 0x1) == 0 { + z.Counts = nil + } + if (zb0001Mask & 0x2) == 0 { + z.Buckets = nil + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *localF64H) Msgsize() (s int) { + s = 1 + 7 + msgp.ArrayHeaderSize + (len(z.Counts) * (msgp.Uint64Size)) + 8 + msgp.ArrayHeaderSize + (len(z.Buckets) * (msgp.Float64Size)) + return +} diff --git a/metrics_gen_test.go b/metrics_gen_test.go index 6e5fdc2..557e99f 100644 --- a/metrics_gen_test.go +++ b/metrics_gen_test.go @@ -1817,6 +1817,119 @@ func BenchmarkDecodeReplicateInfo(b *testing.B) { } } +func TestMarshalUnmarshalRuntimeMetrics(t *testing.T) { + v := RuntimeMetrics{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRuntimeMetrics(b *testing.B) { + v := RuntimeMetrics{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRuntimeMetrics(b *testing.B) { + v := RuntimeMetrics{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRuntimeMetrics(b *testing.B) { + v := RuntimeMetrics{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRuntimeMetrics(t *testing.T) { + v := RuntimeMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRuntimeMetrics Msgsize() is inaccurate") + } + + vn := RuntimeMetrics{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRuntimeMetrics(b *testing.B) { + v := RuntimeMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRuntimeMetrics(b *testing.B) { + v := RuntimeMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + func TestMarshalUnmarshalScannerMetrics(t *testing.T) { v := ScannerMetrics{} bts, err := v.MarshalMsg(nil) @@ -2042,3 +2155,116 @@ func BenchmarkDecodeSiteResyncMetrics(b *testing.B) { } } } + +func TestMarshalUnmarshallocalF64H(t *testing.T) { + v := localF64H{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglocalF64H(b *testing.B) { + v := localF64H{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglocalF64H(b *testing.B) { + v := localF64H{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallocalF64H(b *testing.B) { + v := localF64H{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelocalF64H(t *testing.T) { + v := localF64H{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelocalF64H Msgsize() is inaccurate") + } + + vn := localF64H{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelocalF64H(b *testing.B) { + v := localF64H{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelocalF64H(b *testing.B) { + v := localF64H{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/profiling-commands.go b/profiling-commands.go index 3e6ae2c..7821f2a 100644 --- a/profiling-commands.go +++ b/profiling-commands.go @@ -21,9 +21,7 @@ package madmin import ( "context" - "encoding/json" "errors" - "fmt" "io" "net/http" "net/url" @@ -44,6 +42,7 @@ const ( ProfilerTrace ProfilerType = "trace" // represents Trace profiler type ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type ProfilerGoroutines ProfilerType = "goroutines" // represents Goroutine dumps. + ProfilerRuntime ProfilerType = "runtime" // Include runtime metrics ) // StartProfilingResult holds the result of starting @@ -54,67 +53,6 @@ type StartProfilingResult struct { Error string `json:"error"` } -// StartProfiling makes an admin call to remotely start profiling on a standalone -// server or the whole cluster in case of a distributed setup. -// Deprecated: use Profile API instead -func (adm *AdminClient) StartProfiling(ctx context.Context, profiler ProfilerType) ([]StartProfilingResult, error) { - v := url.Values{} - v.Set("profilerType", string(profiler)) - resp, err := adm.executeMethod(ctx, - http.MethodPost, requestData{ - relPath: adminAPIPrefix + "/profiling/start", - queryValues: v, - }, - ) - defer closeResponse(resp) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, httpRespToErrorResponse(resp) - } - - jsonResult, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - var startResults []StartProfilingResult - err = json.Unmarshal(jsonResult, &startResults) - if err != nil { - return nil, err - } - - return startResults, nil -} - -// DownloadProfilingData makes an admin call to download profiling data of a standalone -// server or of the whole cluster in case of a distributed setup. -// Deprecated: use Profile API instead -func (adm *AdminClient) DownloadProfilingData(ctx context.Context) (io.ReadCloser, error) { - path := fmt.Sprintf(adminAPIPrefix + "/profiling/download") - resp, err := adm.executeMethod(ctx, - http.MethodGet, requestData{ - relPath: path, - }, - ) - if err != nil { - closeResponse(resp) - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, httpRespToErrorResponse(resp) - } - - if resp.Body == nil { - return nil, errors.New("body is nil") - } - - return resp.Body, nil -} - // Profile makes an admin call to remotely start profiling on a standalone // server or the whole cluster in case of a distributed setup for a specified duration. func (adm *AdminClient) Profile(ctx context.Context, profiler ProfilerType, duration time.Duration) (io.ReadCloser, error) { From 2c8d1ef72dc84cf745e9ac1caafa16f30d49895c Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 18 Dec 2024 16:24:13 +0100 Subject: [PATCH 2/4] Sad bc console :/ --- .github/workflows/go.yml | 2 +- .github/workflows/vulncheck.yml | 2 +- metrics.go | 2 +- profiling-commands.go | 63 +++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 1a15cd2..5dfca1f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -47,7 +47,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.22.x] + go-version: [1.23.x] os: [ubuntu-latest, windows-latest, macos-latest] steps: - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml index 1b1e5e5..dbfa15c 100644 --- a/.github/workflows/vulncheck.yml +++ b/.github/workflows/vulncheck.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go-version: [ 1.22.7 ] + go-version: [ 1.23.x ] steps: - name: Check out code into the Go module directory uses: actions/checkout@v4 diff --git a/metrics.go b/metrics.go index c913bc0..3e1d0dd 100644 --- a/metrics.go +++ b/metrics.go @@ -773,7 +773,7 @@ type localF64H struct { } // RuntimeMetrics contains metrics for the go runtime. -// See more at https://pkg.go.dev/runtime/metric +// See more at https://pkg.go.dev/runtime/metrics type RuntimeMetrics struct { // UintMetrics contains KindUint64 values UintMetrics map[string]uint64 `json:"uintMetrics,omitempty"` diff --git a/profiling-commands.go b/profiling-commands.go index 7821f2a..91b957b 100644 --- a/profiling-commands.go +++ b/profiling-commands.go @@ -21,7 +21,9 @@ package madmin import ( "context" + "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -53,6 +55,67 @@ type StartProfilingResult struct { Error string `json:"error"` } +// StartProfiling makes an admin call to remotely start profiling on a standalone +// server or the whole cluster in case of a distributed setup. +// Deprecated: use Profile API instead +func (adm *AdminClient) StartProfiling(ctx context.Context, profiler ProfilerType) ([]StartProfilingResult, error) { + v := url.Values{} + v.Set("profilerType", string(profiler)) + resp, err := adm.executeMethod(ctx, + http.MethodPost, requestData{ + relPath: adminAPIPrefix + "/profiling/start", + queryValues: v, + }, + ) + defer closeResponse(resp) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp) + } + + jsonResult, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var startResults []StartProfilingResult + err = json.Unmarshal(jsonResult, &startResults) + if err != nil { + return nil, err + } + + return startResults, nil +} + +// DownloadProfilingData makes an admin call to download profiling data of a standalone +// server or of the whole cluster in case of a distributed setup. +// Deprecated: use Profile API instead +func (adm *AdminClient) DownloadProfilingData(ctx context.Context) (io.ReadCloser, error) { + path := fmt.Sprintf(adminAPIPrefix + "/profiling/download") + resp, err := adm.executeMethod(ctx, + http.MethodGet, requestData{ + relPath: path, + }, + ) + if err != nil { + closeResponse(resp) + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, httpRespToErrorResponse(resp) + } + + if resp.Body == nil { + return nil, errors.New("body is nil") + } + + return resp.Body, nil +} + // Profile makes an admin call to remotely start profiling on a standalone // server or the whole cluster in case of a distributed setup for a specified duration. func (adm *AdminClient) Profile(ctx context.Context, profiler ProfilerType, duration time.Duration) (io.ReadCloser, error) { From 7b80a8e0f0301782ea4a5e79346adcf0c433046a Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 18 Dec 2024 17:16:39 +0100 Subject: [PATCH 3/4] Make correct maps when merging --- metrics.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metrics.go b/metrics.go index 3e1d0dd..4c9c136 100644 --- a/metrics.go +++ b/metrics.go @@ -794,13 +794,13 @@ func (m *RuntimeMetrics) Merge(other *RuntimeMetrics) { return } if m.UintMetrics == nil { - m.UintMetrics = make(map[string]uint64) + m.UintMetrics = make(map[string]uint64, len(other.UintMetrics)) } if m.FloatMetrics == nil { - m.UintMetrics = make(map[string]uint64) + m.FloatMetrics = make(map[string]float64, len(other.FloatMetrics)) } - if m.FloatMetrics == nil { - m.UintMetrics = make(map[string]uint64) + if m.HistMetrics == nil { + m.HistMetrics = make(map[string]metrics.Float64Histogram, len(other.HistMetrics)) } for k, v := range other.UintMetrics { m.UintMetrics[k] += v From d0467cf8d00acce0cf5ca9c9c540d2c551f96684 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 18 Dec 2024 17:50:29 +0100 Subject: [PATCH 4/4] Fix missing go generate(?) --- metrics.go | 2 +- metrics_gen.go | 105 +++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 9 deletions(-) diff --git a/metrics.go b/metrics.go index 4c9c136..e33e7c8 100644 --- a/metrics.go +++ b/metrics.go @@ -780,8 +780,8 @@ type RuntimeMetrics struct { // FloatMetrics contains KindFloat64 values FloatMetrics map[string]float64 `json:"floatMetrics,omitempty"` - // HistMetrics contains KindFloat64Histogram values + // HistMetrics contains KindFloat64Histogram values HistMetrics map[string]metrics.Float64Histogram `json:"histMetrics,omitempty"` // N tracks the number of merged entries. diff --git a/metrics_gen.go b/metrics_gen.go index 51c04b3..4103848 100644 --- a/metrics_gen.go +++ b/metrics_gen.go @@ -3268,7 +3268,7 @@ func (z *Metrics) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err) return } - var zb0001Mask uint16 /* 9 bits */ + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask for zb0001 > 0 { zb0001-- @@ -3513,6 +3513,25 @@ func (z *Metrics) DecodeMsg(dc *msgp.Reader) (err error) { } } zb0001Mask |= 0x100 + case "go": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Go") + return + } + z.Go = nil + } else { + if z.Go == nil { + z.Go = new(RuntimeMetrics) + } + err = z.Go.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Go") + return + } + } + zb0001Mask |= 0x200 default: err = dc.Skip() if err != nil { @@ -3522,7 +3541,7 @@ func (z *Metrics) DecodeMsg(dc *msgp.Reader) (err error) { } } // Clear omitted fields. - if zb0001Mask != 0x1ff { + if zb0001Mask != 0x3ff { if (zb0001Mask & 0x1) == 0 { z.Scanner = nil } @@ -3550,6 +3569,9 @@ func (z *Metrics) DecodeMsg(dc *msgp.Reader) (err error) { if (zb0001Mask & 0x100) == 0 { z.RPC = nil } + if (zb0001Mask & 0x200) == 0 { + z.Go = nil + } } return } @@ -3557,8 +3579,8 @@ func (z *Metrics) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *Metrics) EncodeMsg(en *msgp.Writer) (err error) { // check for omitted fields - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.Scanner == nil { zb0001Len-- @@ -3596,6 +3618,10 @@ func (z *Metrics) EncodeMsg(en *msgp.Writer) (err error) { zb0001Len-- zb0001Mask |= 0x100 } + if z.Go == nil { + zb0001Len-- + zb0001Mask |= 0x200 + } // variable map header, size zb0001Len err = en.Append(0x80 | uint8(zb0001Len)) if err != nil { @@ -3817,6 +3843,25 @@ func (z *Metrics) EncodeMsg(en *msgp.Writer) (err error) { } } } + if (zb0001Mask & 0x200) == 0 { // if not omitted + // write "go" + err = en.Append(0xa2, 0x67, 0x6f) + if err != nil { + return + } + if z.Go == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Go.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Go") + return + } + } + } } return } @@ -3825,8 +3870,8 @@ func (z *Metrics) EncodeMsg(en *msgp.Writer) (err error) { func (z *Metrics) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // check for omitted fields - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.Scanner == nil { zb0001Len-- @@ -3864,6 +3909,10 @@ func (z *Metrics) MarshalMsg(b []byte) (o []byte, err error) { zb0001Len-- zb0001Mask |= 0x100 } + if z.Go == nil { + zb0001Len-- + zb0001Mask |= 0x200 + } // variable map header, size zb0001Len o = append(o, 0x80|uint8(zb0001Len)) @@ -4001,6 +4050,19 @@ func (z *Metrics) MarshalMsg(b []byte) (o []byte, err error) { } } } + if (zb0001Mask & 0x200) == 0 { // if not omitted + // string "go" + o = append(o, 0xa2, 0x67, 0x6f) + if z.Go == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Go.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Go") + return + } + } + } } return } @@ -4015,7 +4077,7 @@ func (z *Metrics) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - var zb0001Mask uint16 /* 9 bits */ + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask for zb0001 > 0 { zb0001-- @@ -4251,6 +4313,24 @@ func (z *Metrics) UnmarshalMsg(bts []byte) (o []byte, err error) { } } zb0001Mask |= 0x100 + case "go": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Go = nil + } else { + if z.Go == nil { + z.Go = new(RuntimeMetrics) + } + bts, err = z.Go.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Go") + return + } + } + zb0001Mask |= 0x200 default: bts, err = msgp.Skip(bts) if err != nil { @@ -4260,7 +4340,7 @@ func (z *Metrics) UnmarshalMsg(bts []byte) (o []byte, err error) { } } // Clear omitted fields. - if zb0001Mask != 0x1ff { + if zb0001Mask != 0x3ff { if (zb0001Mask & 0x1) == 0 { z.Scanner = nil } @@ -4288,6 +4368,9 @@ func (z *Metrics) UnmarshalMsg(bts []byte) (o []byte, err error) { if (zb0001Mask & 0x100) == 0 { z.RPC = nil } + if (zb0001Mask & 0x200) == 0 { + z.Go = nil + } } o = bts return @@ -4349,6 +4432,12 @@ func (z *Metrics) Msgsize() (s int) { } else { s += z.RPC.Msgsize() } + s += 3 + if z.Go == nil { + s += msgp.NilSize + } else { + s += z.Go.Msgsize() + } return }