Skip to content

Commit

Permalink
tsdb: update vm to v1.76.1; expose more params (pingcap#213) (pingcap…
Browse files Browse the repository at this point in the history 
…#214)

close pingcap#212
  • Loading branch information
@ti-chi-bot
ti-chi-bot authored Nov 30, 2023
1 parent ffccfc1 commit 44845a2
Show file tree
Hide file tree
Showing 7 changed files with 68 additions and 828 deletions.
12 changes: 12 additions & 0 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const (
DefProfileSeconds = 10
DefProfilingTimeoutSeconds = 120
DefProfilingDataRetentionSeconds = 3 * 24 * 60 * 60 // 3 days
DefTSDBRetentionPeriod = "1" // 1 month
DefTSDBSearchMaxUniqueTimeseries = 300000
)

type Config struct {
Expand All @@ -39,6 +41,7 @@ type Config struct {
Storage Storage `toml:"storage" json:"storage"`
ContinueProfiling ContinueProfilingConfig `toml:"-" json:"continuous_profiling"`
Security Security `toml:"security" json:"security"`
TSDB TSDB `toml:"tsdb" json:"tsdb"`
}

var defaultConfig = Config{
Expand All @@ -60,6 +63,10 @@ var defaultConfig = Config{
TimeoutSeconds: DefProfilingTimeoutSeconds,
DataRetentionSeconds: DefProfilingDataRetentionSeconds,
},
TSDB: TSDB{
RetentionPeriod: DefTSDBRetentionPeriod,
SearchMaxUniqueTimeseries: DefTSDBSearchMaxUniqueTimeseries,
},
}

func GetDefaultConfig() Config {
Expand Down Expand Up @@ -388,6 +395,11 @@ func buildTLSConfig(caPath, keyPath, certPath string) *tls.Config {
return tlsConfig
}

type TSDB struct {
RetentionPeriod string `toml:"retention-period" json:"retention_period"`
SearchMaxUniqueTimeseries int64 `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
}

type ContinueProfilingConfig struct {
Enable bool `json:"enable"`
ProfileSeconds int `json:"profile_seconds"`
Expand Down
11 changes: 11 additions & 0 deletions config/config.toml.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,14 @@ path = "data"
ca-path = ""
cert-path = ""
key-path = ""

[tsdb]
# Data with timestamps outside the retentionPeriod is automatically deleted
# The following optional suffixes are supported: h (hour), d (day), w (week), y (year).
# If suffix isn't set, then the duration is counted in months.
retention-period = "1"
# `search-max-unique-timeseries` limits the number of unique time series a single query can find and process.
# VictoriaMetrics(tsdb) keeps in memory some metainformation about the time series located by each query
# and spends some CPU time for processing the found time series. This means that the maximum memory usage
# and CPU usage a single query can use is proportional to `search-max-unique-timeseries`.
search-max-unique-timeseries = 300000
6 changes: 3 additions & 3 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ key-path = "ngm.key"`
data, err := json.Marshal(cfg)
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down Expand Up @@ -114,7 +114,7 @@ path = "data1"`
data, err = json.Marshal(globalCfg)
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))

cfgData = ``
err = ioutil.WriteFile(cfgFileName, []byte(cfgData), 0666)
Expand All @@ -125,7 +125,7 @@ path = "data1"`
data, err = json.Marshal(GetGlobalConfig())
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
}

func TestConfigValid(t *testing.T) {
Expand Down
11 changes: 3 additions & 8 deletions database/timeseries/vm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package timeseries

import (
"flag"
"fmt"
"os"
"path"
"time"
Expand All @@ -14,30 +15,24 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/pingcap/log"
"github.com/spf13/pflag"
"go.uber.org/zap"
)

var (
retentionPeriod = pflag.String("retention-period", "1", "Data with timestamps outside the retentionPeriod is automatically deleted\nThe following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months")
)

func Init(cfg *config.Config) {
if err := initLogger(cfg); err != nil {
log.Fatal("Failed to open log file", zap.Error(err))
}
initDataDir(path.Join(cfg.Storage.Path, "tsdb"))

_ = flag.Set("retentionPeriod", *retentionPeriod)
_ = flag.Set("retentionPeriod", cfg.TSDB.RetentionPeriod)
_ = flag.Set("search.maxStepForPointsAdjustment", "1s")
_ = flag.Set("search.maxUniqueTimeseries", fmt.Sprintf("%d", cfg.TSDB.SearchMaxUniqueTimeseries))

// Some components in VictoriaMetrics want parsed arguments, i.e. assert `flag.Parsed()`. Make them happy.
_ = flag.CommandLine.Parse(nil)

startTime := time.Now()
storage.SetMinScrapeIntervalForDeduplication(0)
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
vmselect.Init()
vminsert.Init()
Expand Down
18 changes: 9 additions & 9 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.19

require (
github.com/BurntSushi/toml v1.2.1
github.com/VictoriaMetrics/VictoriaMetrics v1.65.0
github.com/VictoriaMetrics/VictoriaMetrics v1.76.1
github.com/dgraph-io/badger/v3 v3.2103.1
github.com/genjidb/genji v0.13.0
github.com/genjidb/genji/engine/badgerengine v0.13.0
Expand All @@ -25,7 +25,7 @@ require (
github.com/prometheus/common v0.39.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.3
github.com/valyala/gozstd v1.14.2
github.com/valyala/gozstd v1.16.0
github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f
go.etcd.io/etcd/api/v3 v3.5.7
go.etcd.io/etcd/client/pkg/v3 v3.5.10
Expand All @@ -42,10 +42,10 @@ require (
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/VictoriaMetrics/fasthttp v1.0.16 // indirect
github.com/VictoriaMetrics/metrics v1.17.3 // indirect
github.com/VictoriaMetrics/metricsql v0.21.0 // indirect
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
github.com/VictoriaMetrics/fasthttp v1.1.0 // indirect
github.com/VictoriaMetrics/metrics v1.18.1 // indirect
github.com/VictoriaMetrics/metricsql v0.41.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
Expand Down Expand Up @@ -135,10 +135,10 @@ require (
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fastjson v1.6.3 // indirect
github.com/valyala/fastrand v1.0.0 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/valyala/histogram v1.1.2 // indirect
github.com/valyala/quicktemplate v1.6.3 // indirect
github.com/valyala/histogram v1.2.0 // indirect
github.com/valyala/quicktemplate v1.7.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
Expand Down
Loading

0 comments on commit 44845a2

Please sign in to comment.