From 7de818ff3da08077872dd1df6669a8f720ad91e3 Mon Sep 17 00:00:00 2001 From: Vishnu Challa Date: Wed, 3 May 2023 05:47:11 -0400 Subject: [PATCH] Integrating indexers from go-commons (#296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * integrating indexers in go-commons * modifying go version to the latest * fixing linting erros * making it compatible with go-commons indexers * fixing index type casting issue * fix linting erros * Update pkg/measurements/pod_latency.go Co-authored-by: Raรบl Sevilla * Update pkg/measurements/vmi_latency.go Co-authored-by: Raรบl Sevilla --------- Co-authored-by: Raรบl Sevilla --- cmd/kube-burner/kube-burner.go | 14 ++-- cmd/kube-burner/ocp.go | 11 ++- go.mod | 8 +- go.sum | 12 ++- pkg/alerting/alert_manager.go | 16 ++-- pkg/burner/job.go | 11 +-- pkg/burner/metadata.go | 12 ++- pkg/config/config.go | 3 +- pkg/config/types.go | 35 +-------- pkg/indexers/elastic.go | 135 -------------------------------- pkg/indexers/factory.go | 52 ------------ pkg/indexers/local.go | 65 --------------- pkg/measurements/factory.go | 2 +- pkg/measurements/pod_latency.go | 21 ++++- pkg/measurements/vmi_latency.go | 21 ++++- pkg/prometheus/prometheus.go | 14 +++- pkg/util/metrics/metrics.go | 13 +-- pkg/util/metrics/tarball.go | 13 +-- pkg/util/metrics/types.go | 2 +- pkg/util/metrics/utils.go | 2 +- pkg/workloads/helpers.go | 10 ++- 21 files changed, 132 insertions(+), 340 deletions(-) delete mode 100644 pkg/indexers/elastic.go delete mode 100644 pkg/indexers/factory.go delete mode 100644 pkg/indexers/local.go diff --git a/cmd/kube-burner/kube-burner.go b/cmd/kube-burner/kube-burner.go index 10841d021..b91bcf8ea 100644 --- a/cmd/kube-burner/kube-burner.go +++ b/cmd/kube-burner/kube-burner.go @@ -32,7 +32,7 @@ import ( "github.com/cloud-bulldozer/kube-burner/pkg/util/metrics" "github.com/cloud-bulldozer/kube-burner/pkg/version" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" uid "github.com/satori/go.uuid" @@ -250,11 +250,13 @@ func importCmd() *cobra.Command { if err != nil { log.Fatal(err.Error()) } - indexer, err := indexers.NewIndexer(configSpec) + indexerConfig := configSpec.GlobalConfig.IndexerConfig + log.Infof("๐Ÿ“ Creating indexer: %s", indexerConfig.Type) + indexer, err := indexers.NewIndexer(indexerConfig) if err != nil { log.Fatal(err.Error()) } - err = metrics.ImportTarball(tarball, indexer) + err = metrics.ImportTarball(tarball, indexer, indexerConfig.MetricsDirectory) if err != nil { log.Fatal(err.Error()) } @@ -291,7 +293,9 @@ func alertCmd() *cobra.Command { } } if configSpec.GlobalConfig.IndexerConfig.Enabled { - indexer, err = indexers.NewIndexer(configSpec) + indexerConfig := configSpec.GlobalConfig.IndexerConfig + log.Infof("๐Ÿ“ Creating indexer: %s", indexerConfig.Type) + indexer, err = indexers.NewIndexer(indexerConfig) if err != nil { log.Fatal(err.Error()) } @@ -308,7 +312,7 @@ func alertCmd() *cobra.Command { } startTime := time.Unix(start, 0) endTime := time.Unix(end, 0) - if alertM, err = alerting.NewAlertManager(alertProfile, uuid, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p); err != nil { + if alertM, err = alerting.NewAlertManager(alertProfile, uuid, configSpec.GlobalConfig.IndexerConfig.Index, indexer, p); err != nil { log.Fatalf("Error creating alert manager: %s", err) } rc := alertM.Evaluate(startTime, endTime) diff --git a/cmd/kube-burner/ocp.go b/cmd/kube-burner/ocp.go index 1d4bdc850..f1c454458 100644 --- a/cmd/kube-burner/ocp.go +++ b/cmd/kube-burner/ocp.go @@ -22,12 +22,11 @@ import ( "strings" "time" - log "github.com/sirupsen/logrus" - - "github.com/cloud-bulldozer/kube-burner/pkg/config" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/discovery" "github.com/cloud-bulldozer/kube-burner/pkg/workloads" uid "github.com/satori/go.uuid" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -41,7 +40,7 @@ func openShiftCmd() *cobra.Command { Long: `This subcommand is meant to be used against OpenShift clusters and serve as a shortcut to trigger well-known workloads`, } var wh workloads.WorkloadHelper - var indexingType config.IndexerType + var indexingType indexers.IndexerType var indexing bool esServer := ocpCmd.PersistentFlags().String("es-server", "", "Elastic Search endpoint") localIndexing := ocpCmd.PersistentFlags().Bool("local-indexing", false, "Enable local indexing") @@ -62,9 +61,9 @@ func openShiftCmd() *cobra.Command { if *esServer != "" || *localIndexing { indexing = true if *esServer != "" { - indexingType = config.ElasticIndexer + indexingType = indexers.ElasticIndexer } else { - indexingType = config.LocalIndexer + indexingType = indexers.LocalIndexer } } envVars := map[string]string{ diff --git a/go.mod b/go.mod index 64b0d328f..2baf777f8 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,11 @@ go 1.19 require ( github.com/Masterminds/sprig/v3 v3.2.2 - github.com/elastic/go-elasticsearch/v7 v7.13.1 + github.com/cloud-bulldozer/go-commons v1.0.0 github.com/prometheus/client_golang v1.15.0 github.com/prometheus/common v0.42.0 github.com/satori/go.uuid v1.2.0 - github.com/sirupsen/logrus v1.8.1 - github.com/spf13/cast v1.3.1 + github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.6.1 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 gopkg.in/yaml.v3 v3.0.1 @@ -34,6 +33,7 @@ require ( github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -58,10 +58,12 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opensearch-project/opensearch-go v1.1.0 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/pborman/uuid v1.2.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect golang.org/x/net v0.7.0 // indirect diff --git a/go.sum b/go.sum index 547808669..3aa396dae 100644 --- a/go.sum +++ b/go.sum @@ -92,6 +92,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.42.27/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -113,6 +114,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloud-bulldozer/go-commons v1.0.0 h1:YOWLH/aN/PN5MDaVrhktzeTM1SsU8XHIHfdKjwLLM8c= +github.com/cloud-bulldozer/go-commons v1.0.0/go.mod h1:HLdZZlRpv8bWVkq1g9AyLE7bpsbAbau5Uqfw/fM9bLE= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -334,6 +337,8 @@ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -428,6 +433,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= +github.com/opensearch-project/opensearch-go v1.1.0 h1:eG5sh3843bbU1itPRjA9QXbxcg8LaZ+DjEzQH9aLN3M= +github.com/opensearch-project/opensearch-go v1.1.0/go.mod h1:+6/XHCuTH+fwsMJikZEWsucZ4eZMma3zNSeLrTtVGbo= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -490,8 +497,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -674,6 +682,7 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -779,6 +788,7 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/alerting/alert_manager.go b/pkg/alerting/alert_manager.go index ac8c98bc6..a7af394a1 100644 --- a/pkg/alerting/alert_manager.go +++ b/pkg/alerting/alert_manager.go @@ -22,12 +22,11 @@ import ( "text/template" "time" - "github.com/prometheus/common/model" - log "github.com/sirupsen/logrus" - - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" "github.com/cloud-bulldozer/kube-burner/pkg/util" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) @@ -205,5 +204,12 @@ func parseMatrix(value model.Value, description string, severity severityLevel) func (a *AlertManager) index(alertSet []interface{}) { log.Info("Indexing alerts") - (*a.indexer).Index(alertSet, indexers.IndexingOpts{MetricName: alertMetricName}) + log.Infof("Indexing metric %s", alertMetricName) + log.Debugf("Indexing [%d] documents", len(alertSet)) + resp, err := (*a.indexer).Index(alertSet, indexers.IndexingOpts{MetricName: alertMetricName}) + if err != nil { + log.Error(err) + } else { + log.Info(resp) + } } diff --git a/pkg/burner/job.go b/pkg/burner/job.go index c029dc3f8..9ad720ec5 100644 --- a/pkg/burner/job.go +++ b/pkg/burner/job.go @@ -19,9 +19,9 @@ import ( "fmt" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/alerting" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/measurements" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" "github.com/cloud-bulldozer/kube-burner/pkg/util" @@ -81,6 +81,7 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr var rc int var prometheusJobList []prometheus.Job res := make(chan int, 1) + globalConfig := configSpec.GlobalConfig log.Infof("๐Ÿ”ฅ Starting kube-burner (%s@%s) with UUID %s", version.Version, version.GitCommit, uuid) go func() { var innerRC int @@ -159,14 +160,14 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr } log.Infof("Job %s took %.2f seconds", job.Config.Name, elapsedTime) } - if configSpec.GlobalConfig.IndexerConfig.Enabled { + if globalConfig.IndexerConfig.Enabled { for _, job := range jobList { elapsedTime := job.End.Sub(job.Start).Seconds() indexjobSummaryInfo(indexer, uuid, elapsedTime, job.Config, job.Start, metadata) } } // We initialize cleanup as soon as the benchmark finishes - if configSpec.GlobalConfig.GC { + if globalConfig.GC { go CleanupNamespaces(context.TODO(), v1.ListOptions{LabelSelector: fmt.Sprintf("kube-burner-uuid=%v", uuid)}, false) } for idx, prometheusClient := range prometheusClients { @@ -178,7 +179,7 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr } prometheusClient.JobList = prometheusJobList // If prometheus is enabled query metrics from the start of the first job to the end of the last one - if configSpec.GlobalConfig.IndexerConfig.Enabled { + if globalConfig.IndexerConfig.Enabled { metrics.ScrapeMetrics(prometheusClient, indexer) metrics.HandleTarball(configSpec) } @@ -192,7 +193,7 @@ func Run(configSpec config.Spec, uuid string, prometheusClients []*prometheus.Pr log.Errorf("%v timeout reached", timeout) rc = rcTimeout } - if configSpec.GlobalConfig.GC { + if globalConfig.GC { // Use timeout/4 to garbage collect namespaces ctx, cancel := context.WithTimeout(context.Background(), timeout/4) defer cancel() diff --git a/pkg/burner/metadata.go b/pkg/burner/metadata.go index 0cbba37f5..10a5fe053 100644 --- a/pkg/burner/metadata.go +++ b/pkg/burner/metadata.go @@ -18,9 +18,10 @@ import ( "fmt" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/version" + log "github.com/sirupsen/logrus" ) type jobSummary struct { @@ -48,5 +49,12 @@ func indexjobSummaryInfo(indexer *indexers.Indexer, uuid string, elapsedTime flo Version: fmt.Sprintf("%v@%v", version.Version, version.GitCommit), }, } - (*indexer).Index(metadataInfo, indexers.IndexingOpts{MetricName: jobSummaryMetric, JobName: jobConfig.Name}) + log.Infof("Indexing metric %s", jobSummaryMetric) + log.Debugf("Indexing [%d] documents", len(metadataInfo)) + resp, err := (*indexer).Index(metadataInfo, indexers.IndexingOpts{MetricName: jobSummaryMetric, JobName: jobConfig.Name}) + if err != nil { + log.Error(err) + } else { + log.Info(resp) + } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 4ad0f1b91..57167e7bb 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -23,6 +23,7 @@ import ( "path/filepath" "time" + "github.com/cloud-bulldozer/go-commons/indexers" mtypes "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" "github.com/cloud-bulldozer/kube-burner/pkg/util" log "github.com/sirupsen/logrus" @@ -42,7 +43,7 @@ var configSpec = Spec{ GC: false, RequestTimeout: 15 * time.Second, Measurements: []mtypes.Measurement{}, - IndexerConfig: IndexerConfig{ + IndexerConfig: indexers.IndexerConfig{ Enabled: false, InsecureSkipVerify: false, MetricsDirectory: "collected-metrics", diff --git a/pkg/config/types.go b/pkg/config/types.go index ceb2e58c5..a3d83a88d 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -17,6 +17,7 @@ package config import ( "time" + "github.com/cloud-bulldozer/go-commons/indexers" mtypes "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" ) @@ -32,16 +33,6 @@ const ( PatchJob JobType = "patch" ) -// IndexerType type of indexer -type IndexerType string - -const ( - // Elastic indexer that send metrics to the configures ES instance - ElasticIndexer IndexerType = "elastic" - // Local indexer that writes metrics to local directory - LocalIndexer IndexerType = "local" -) - // Spec configuration root type Spec struct { // GlobalConfig defines global configuration parameters @@ -50,32 +41,10 @@ type Spec struct { Jobs []Job `yaml:"jobs"` } -// IndexerConfig holds the indexer configuration -type IndexerConfig struct { - // Type type of indexer - Type IndexerType `yaml:"type"` - // ESServers List of ElasticSearch instances - ESServers []string `yaml:"esServers"` - // DefaultIndex default index to send prometheus metrics - DefaultIndex string `yaml:"defaultIndex"` - // Port indexer port - Port int `yaml:"port"` - // InsecureSkipVerify disable TLS ceriticate verification - InsecureSkipVerify bool `yaml:"insecureSkipVerify"` - // Enabled enable indexer - Enabled bool `yaml:"enabled"` - // Directory to save metrics files in - MetricsDirectory string `yaml:"metricsDirectory"` - // Create tarball - CreateTarball bool `yaml:"createTarball"` - // TarBall name - TarballName string `yaml:"tarballName"` -} - // GlobalConfig holds the global configuration type GlobalConfig struct { // IndexerConfig contains a IndexerConfig definition - IndexerConfig IndexerConfig `yaml:"indexerConfig"` + IndexerConfig indexers.IndexerConfig `yaml:"indexerConfig"` // Measurements describes a list of measurements kube-burner // will take along with job Measurements []mtypes.Measurement `yaml:"measurements"` diff --git a/pkg/indexers/elastic.go b/pkg/indexers/elastic.go deleted file mode 100644 index 6fad2a85b..000000000 --- a/pkg/indexers/elastic.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright 2020 The Kube-burner Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexers - -import ( - "bytes" - "context" - "crypto/sha256" - "crypto/tls" - "encoding/hex" - "encoding/json" - "fmt" - "net/http" - "runtime" - "strings" - "sync" - "time" - - "github.com/cloud-bulldozer/kube-burner/pkg/config" - log "github.com/sirupsen/logrus" - - elasticsearch "github.com/elastic/go-elasticsearch/v7" - "github.com/elastic/go-elasticsearch/v7/esutil" -) - -const elastic = "elastic" - -// Elastic ElasticSearch instance -type Elastic struct { - client *elasticsearch.Client - index string -} - -func init() { - indexerMap[elastic] = &Elastic{} -} - -func (esIndexer *Elastic) new(configSpec config.Spec) error { - esConfig := configSpec.GlobalConfig.IndexerConfig - if esConfig.DefaultIndex == "" { - return fmt.Errorf("index name not specified") - } - esIndex := strings.ToLower(esConfig.DefaultIndex) - cfg := elasticsearch.Config{ - Addresses: esConfig.ESServers, - Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: esConfig.InsecureSkipVerify}}, - } - ESClient, err := elasticsearch.NewClient(cfg) - if err != nil { - return fmt.Errorf("error creating the ES client: %s", err) - } - r, err := ESClient.Cluster.Health() - if err != nil { - return fmt.Errorf("ES health check failed: %s", err) - } - if r.StatusCode != 200 { - return fmt.Errorf("unexpected ES status code: %d", r.StatusCode) - } - esIndexer.client = ESClient - esIndexer.index = esIndex - r, _ = esIndexer.client.Indices.Exists([]string{esIndex}) - if r.IsError() { - log.Infof("Creating index %s", esIndex) - r, _ = esIndexer.client.Indices.Create(esIndex) - if r.IsError() { - return fmt.Errorf("error creating index %s on ES: %s", esIndex, r.String()) - } - } - return nil -} - -// Index uses bulkIndexer to index the documents in the given index -func (esIndexer *Elastic) Index(documents []interface{}, opts IndexingOpts) { - var statString string - var indexerStatsLock sync.Mutex - log.Infof("Indexing metric %s", opts.MetricName) - indexerStats := make(map[string]int) - hasher := sha256.New() - bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ - Client: esIndexer.client, - Index: esIndexer.index, - FlushBytes: 5e+6, - NumWorkers: runtime.NumCPU(), - Timeout: 10 * time.Minute, // TODO: hardcoded - }) - if err != nil { - log.Errorf("Error creating the indexer: %s", err) - } - start := time.Now().UTC() - log.Debugf("Indexing [%d] documents in %s", len(documents), esIndexer.index) - for _, document := range documents { - j, err := json.Marshal(document) - if err != nil { - log.Errorf("Cannot encode document %s: %s", document, err) - } - hasher.Write(j) - err = bi.Add( - context.Background(), - esutil.BulkIndexerItem{ - Action: "index", - Body: bytes.NewReader(j), - DocumentID: hex.EncodeToString(hasher.Sum(nil)), - OnSuccess: func(c context.Context, bii esutil.BulkIndexerItem, biri esutil.BulkIndexerResponseItem) { - indexerStatsLock.Lock() - defer indexerStatsLock.Unlock() - indexerStats[biri.Result]++ - }, - }, - ) - if err != nil { - log.Errorf("Unexpected ES indexing error: %s", err) - } - hasher.Reset() - } - if err := bi.Close(context.Background()); err != nil { - log.Fatalf("Unexpected ES error: %s", err) - } - dur := time.Since(start) - for stat, val := range indexerStats { - statString += fmt.Sprintf(" %s=%d", stat, val) - } - log.Debugf("Indexing finished in %v:%v", dur.Truncate(time.Millisecond), statString) -} diff --git a/pkg/indexers/factory.go b/pkg/indexers/factory.go deleted file mode 100644 index 5dfa3be26..000000000 --- a/pkg/indexers/factory.go +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 The Kube-burner Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package indexers - -import ( - "fmt" - - "github.com/cloud-bulldozer/kube-burner/pkg/config" - log "github.com/sirupsen/logrus" -) - -// Indexer indexer interface -type Indexer interface { - Index([]interface{}, IndexingOpts) - new(config.Spec) error -} - -type IndexingOpts struct { - MetricName string - JobName string -} - -var indexerMap = make(map[config.IndexerType]Indexer) - -// NewIndexer creates a new Indexer with the specified IndexerConfig -func NewIndexer(configSpec config.Spec) (*Indexer, error) { - var indexer Indexer - var exists bool - cfg := configSpec.GlobalConfig.IndexerConfig - if indexer, exists = indexerMap[cfg.Type]; exists { - log.Infof("๐Ÿ“ Creating indexer: %s", cfg.Type) - err := indexer.new(configSpec) - if err != nil { - return &indexer, err - } - } else { - return &indexer, fmt.Errorf("Indexer not found: %s", cfg.Type) - } - return &indexer, nil -} diff --git a/pkg/indexers/local.go b/pkg/indexers/local.go deleted file mode 100644 index 5b4a809ba..000000000 --- a/pkg/indexers/local.go +++ /dev/null @@ -1,65 +0,0 @@ -package indexers - -import ( - "encoding/json" - "fmt" - "os" - "path" - - "github.com/cloud-bulldozer/kube-burner/pkg/config" - log "github.com/sirupsen/logrus" -) - -// Copyright 2023 The Kube-burner Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -const local = "local" - -type Local struct { - metricsDirectory string -} - -func init() { - indexerMap[local] = &Local{} -} - -func (l *Local) new(configSpec config.Spec) error { - if configSpec.GlobalConfig.IndexerConfig.MetricsDirectory == "" { - return fmt.Errorf("directory name not specified") - } - l.metricsDirectory = configSpec.GlobalConfig.IndexerConfig.MetricsDirectory - err := os.MkdirAll(l.metricsDirectory, 0744) - return err -} - -// Index uses generates a local file with the given name and metrics -func (l *Local) Index(documents []interface{}, opts IndexingOpts) { - var metricName string - if opts.JobName != "" { - metricName = fmt.Sprintf("%s-%s.json", opts.MetricName, opts.JobName) - } else { - metricName = fmt.Sprintf("%s.json", opts.MetricName) - } - filename := path.Join(l.metricsDirectory, metricName) - log.Infof("Writing metric to: %s", filename) - f, err := os.Create(filename) - if err != nil { - log.Errorf("Error creating metrics file %s: %s", filename, err) - } - defer f.Close() - jsonEnc := json.NewEncoder(f) - if jsonEnc.Encode(documents); err != nil { - log.Errorf("JSON encoding error: %s", err) - } -} diff --git a/pkg/measurements/factory.go b/pkg/measurements/factory.go index 1e60cdb42..56ecd2603 100644 --- a/pkg/measurements/factory.go +++ b/pkg/measurements/factory.go @@ -18,8 +18,8 @@ import ( "fmt" "sync" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" log "github.com/sirupsen/logrus" "k8s.io/client-go/kubernetes" diff --git a/pkg/measurements/pod_latency.go b/pkg/measurements/pod_latency.go index 1a10356c0..4e2eeeac5 100644 --- a/pkg/measurements/pod_latency.go +++ b/pkg/measurements/pod_latency.go @@ -22,8 +22,8 @@ import ( "sync" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/measurements/metrics" "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" log "github.com/sirupsen/logrus" @@ -176,8 +176,23 @@ func (p *podLatency) stop() (int, error) { // index sends metrics to the configured indexer func (p *podLatency) index() { - (*factory.indexer).Index(p.normLatencies, indexers.IndexingOpts{MetricName: podLatencyMeasurement, JobName: factory.jobConfig.Name}) - (*factory.indexer).Index(p.latencyQuantiles, indexers.IndexingOpts{MetricName: podLatencyQuantilesMeasurement, JobName: factory.jobConfig.Name}) + indexingOpts := indexers.IndexingOpts{ + JobName: factory.jobConfig.Name, + } + metricMap := map[string][]interface{}{ + podLatencyMeasurement: p.normLatencies, + podLatencyQuantilesMeasurement: p.latencyQuantiles, + } + for metricName, data := range metricMap { + indexingOpts.MetricName = metricName + log.Debugf("Indexing [%d] documents", len(data)) + resp, err := (*factory.indexer).Index(data, indexingOpts) + if err != nil { + log.Error(err.Error()) + } else { + log.Info(resp) + } + } } func (p *podLatency) normalizeMetrics() { diff --git a/pkg/measurements/vmi_latency.go b/pkg/measurements/vmi_latency.go index b0533a606..dc97059e8 100644 --- a/pkg/measurements/vmi_latency.go +++ b/pkg/measurements/vmi_latency.go @@ -22,7 +22,7 @@ import ( "sync" "time" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/measurements/metrics" "github.com/cloud-bulldozer/kube-burner/pkg/measurements/types" log "github.com/sirupsen/logrus" @@ -379,8 +379,23 @@ func (p *vmiLatency) stop() (int, error) { // index sends metrics to the configured indexer func (p *vmiLatency) index() { - (*factory.indexer).Index(p.normLatencies, indexers.IndexingOpts{MetricName: podLatencyMeasurement, JobName: factory.jobConfig.Name}) - (*factory.indexer).Index(p.latencyQuantiles, indexers.IndexingOpts{MetricName: podLatencyQuantilesMeasurement, JobName: factory.jobConfig.Name}) + indexingOpts := indexers.IndexingOpts{ + JobName: factory.jobConfig.Name, + } + metricMap := map[string][]interface{}{ + podLatencyMeasurement: p.normLatencies, + podLatencyQuantilesMeasurement: p.latencyQuantiles, + } + for metricName, data := range metricMap { + indexingOpts.MetricName = metricName + log.Debugf("Indexing [%d] documents", len(data)) + resp, err := (*factory.indexer).Index(data, indexingOpts) + if err != nil { + log.Error(err.Error()) + } else { + log.Info(resp) + } + } } func (p *vmiLatency) normalizeMetrics() { diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go index b074a80b8..93bb39613 100644 --- a/pkg/prometheus/prometheus.go +++ b/pkg/prometheus/prometheus.go @@ -24,8 +24,8 @@ import ( "text/template" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/util" api "github.com/prometheus/client_golang/api" apiv1 "github.com/prometheus/client_golang/api/prometheus/v1" @@ -140,8 +140,16 @@ func (p *Prometheus) ScrapeJobsMetrics(indexer *indexers.Indexer) error { continue } } - if p.ConfigSpec.GlobalConfig.IndexerConfig.Enabled { - (*indexer).Index(datapoints, indexers.IndexingOpts{MetricName: md.MetricName}) + indexerConfig := p.ConfigSpec.GlobalConfig.IndexerConfig + if indexerConfig.Enabled { + log.Infof("Indexing metric %s", md.MetricName) + log.Debugf("Indexing [%d] documents", len(datapoints)) + resp, err := (*indexer).Index(datapoints, indexers.IndexingOpts{MetricName: md.MetricName}) + if err != nil { + log.Error(err.Error()) + } else { + log.Info(resp) + } } } return nil diff --git a/pkg/util/metrics/metrics.go b/pkg/util/metrics/metrics.go index c537a1468..5ba940f9c 100644 --- a/pkg/util/metrics/metrics.go +++ b/pkg/util/metrics/metrics.go @@ -17,9 +17,8 @@ package metrics import ( "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/alerting" - "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" "github.com/cloud-bulldozer/kube-burner/pkg/util" log "github.com/sirupsen/logrus" @@ -36,9 +35,11 @@ func ProcessMetricsScraperConfig(metricsScraperConfig ScraperConfig) Scraper { var alertM *alerting.AlertManager userMetadataContent := make(map[string]interface{}) if configSpec.GlobalConfig.IndexerConfig.Enabled { - indexer, err = indexers.NewIndexer(configSpec) + indexerConfig := configSpec.GlobalConfig.IndexerConfig + log.Infof("๐Ÿ“ Creating indexer: %s", indexerConfig.Type) + indexer, err = indexers.NewIndexer(indexerConfig) if err != nil { - log.Fatalf("%v indexer: %v", configSpec.GlobalConfig.IndexerConfig.Type, err.Error()) + log.Fatalf("%v indexer: %v", indexerConfig.Type, err.Error()) } } if metricsScraperConfig.UserMetaData != "" { @@ -90,14 +91,14 @@ func ProcessMetricsScraperConfig(metricsScraperConfig ScraperConfig) Scraper { }, } ScrapeMetrics(p, indexer) - if configSpec.GlobalConfig.IndexerConfig.Type == config.LocalIndexer { + if configSpec.GlobalConfig.IndexerConfig.Type == indexers.LocalIndexer { HandleTarball(configSpec) } } else { updateParamIfEmpty(&metricsEndpoint.AlertProfile, metricsScraperConfig.AlertProfile) updateParamIfEmpty(&metricsEndpoint.AlertProfile, configSpec.GlobalConfig.AlertProfile) if metricsEndpoint.AlertProfile != "" { - if alertM, err = alerting.NewAlertManager(metricsEndpoint.AlertProfile, metricsScraperConfig.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p); err != nil { + if alertM, err = alerting.NewAlertManager(metricsEndpoint.AlertProfile, metricsScraperConfig.UUID, configSpec.GlobalConfig.IndexerConfig.Index, indexer, p); err != nil { log.Fatalf("Error creating alert manager: %s", err) } } diff --git a/pkg/util/metrics/tarball.go b/pkg/util/metrics/tarball.go index fce2a668a..8d84372e8 100644 --- a/pkg/util/metrics/tarball.go +++ b/pkg/util/metrics/tarball.go @@ -25,12 +25,11 @@ import ( "os" "path/filepath" - "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" + "github.com/cloud-bulldozer/go-commons/indexers" log "github.com/sirupsen/logrus" ) -func createTarball(indexerConfig config.IndexerConfig) error { +func createTarball(indexerConfig indexers.IndexerConfig) error { tarball, err := os.Create(fmt.Sprintf(indexerConfig.TarballName)) if err != nil { return fmt.Errorf("Could not create tarball file: %v", err) @@ -68,7 +67,7 @@ func createTarball(indexerConfig config.IndexerConfig) error { return nil } -func ImportTarball(tarball string, indexer *indexers.Indexer) error { +func ImportTarball(tarball string, indexer *indexers.Indexer, metricsDir string) error { log.Infof("Importing tarball %v", tarball) var rawData bytes.Buffer tarballFile, err := os.Open(tarball) @@ -95,7 +94,11 @@ func ImportTarball(tarball string, indexer *indexers.Indexer) error { return fmt.Errorf("Tarball read error: %v", err) } log.Infof("Importing metrics from %s", hdr.Name) - (*indexer).Index(metrics, indexers.IndexingOpts{}) + log.Infof("Writing metric to: %s", metricsDir) + _, err = (*indexer).Index(metrics, indexers.IndexingOpts{}) + if err != nil { + return err + } } return nil } diff --git a/pkg/util/metrics/types.go b/pkg/util/metrics/types.go index 659a44218..81e1f8c6b 100644 --- a/pkg/util/metrics/types.go +++ b/pkg/util/metrics/types.go @@ -17,9 +17,9 @@ package metrics import ( "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/alerting" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" ) diff --git a/pkg/util/metrics/utils.go b/pkg/util/metrics/utils.go index 9355f0e8b..625cbe3a7 100644 --- a/pkg/util/metrics/utils.go +++ b/pkg/util/metrics/utils.go @@ -19,8 +19,8 @@ import ( "io" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/config" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" "github.com/cloud-bulldozer/kube-burner/pkg/util" log "github.com/sirupsen/logrus" diff --git a/pkg/workloads/helpers.go b/pkg/workloads/helpers.go index 5c7e1de1f..92d97d782 100644 --- a/pkg/workloads/helpers.go +++ b/pkg/workloads/helpers.go @@ -26,11 +26,11 @@ import ( "path" "time" + "github.com/cloud-bulldozer/go-commons/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/alerting" "github.com/cloud-bulldozer/kube-burner/pkg/burner" "github.com/cloud-bulldozer/kube-burner/pkg/config" "github.com/cloud-bulldozer/kube-burner/pkg/discovery" - "github.com/cloud-bulldozer/kube-burner/pkg/indexers" "github.com/cloud-bulldozer/kube-burner/pkg/prometheus" "github.com/cloud-bulldozer/kube-burner/pkg/util" "github.com/cloud-bulldozer/kube-burner/pkg/util/metrics" @@ -216,9 +216,11 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { log.Fatal(err) } if wh.indexing { - indexer, err = indexers.NewIndexer(configSpec) + indexerConfig := configSpec.GlobalConfig.IndexerConfig + log.Infof("๐Ÿ“ Creating indexer: %s", indexerConfig.Type) + indexer, err = indexers.NewIndexer(indexerConfig) if err != nil { - log.Fatalf("%v indexer: %v", configSpec.GlobalConfig.IndexerConfig.Type, err.Error()) + log.Fatalf("%v indexer: %v", indexerConfig.Type, err.Error()) } } if wh.metricsEndpoint != "" { @@ -241,7 +243,7 @@ func (wh *WorkloadHelper) run(workload, metricsProfile string) { log.Fatal(err) } if wh.alerting && configSpec.GlobalConfig.AlertProfile != "" { - alertM, err = alerting.NewAlertManager(configSpec.GlobalConfig.AlertProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.DefaultIndex, indexer, p) + alertM, err = alerting.NewAlertManager(configSpec.GlobalConfig.AlertProfile, wh.Metadata.UUID, configSpec.GlobalConfig.IndexerConfig.Index, indexer, p) if err != nil { log.Fatal(err) }