From 14cea77a4ceb5bbc07e31f9713808c37de7851c1 Mon Sep 17 00:00:00 2001 From: Jmnote Date: Wed, 5 Apr 2023 12:34:33 +0900 Subject: [PATCH] bugfix loading config & test code (#20) --- .github/workflows/ci.yml | 18 +------- Makefile | 5 ++- etc/.gitignore | 1 - ...asources.example.yaml => datasources.yaml} | 4 -- pkg/configuration/config.go | 30 ++------------ pkg/configuration/config_test.go | 21 +++++++++- pkg/handler/alert.go | 4 +- pkg/handler/dashboard.go | 5 ++- pkg/server.go | 5 ++- pkg/store/alertRuleStore.go | 41 +++++++++++++++++++ pkg/store/alertRuleStore_test.go | 26 ++++++++++++ pkg/store/alertrulestore.go | 41 ------------------- pkg/store/userStore.go | 7 ++-- scripts/checks.sh | 9 ++++ scripts/test-cover.sh | 2 +- 15 files changed, 117 insertions(+), 102 deletions(-) delete mode 100644 etc/.gitignore rename etc/{datasources.example.yaml => datasources.yaml} (57%) create mode 100644 pkg/store/alertRuleStore.go create mode 100644 pkg/store/alertRuleStore_test.go delete mode 100644 pkg/store/alertrulestore.go create mode 100755 scripts/checks.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ca1b26..5bcc2dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - args: --timeout 3m - + args: --timeout 5m go-test-failfast: runs-on: ubuntu-latest @@ -65,22 +64,9 @@ jobs: - uses: actions/setup-go@v4 - uses: actions/checkout@v3 - name: go test with coverage - #run: go test -v -covermode=count -coverprofile=coverage.out - run: go test ./... -race -coverprofile=coverage.out -covermode=atomic + run: go test ./... -v -race -coverprofile=coverage.out - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 - #- name: Convert coverage to lcov - # uses: jandelgado/gcov2lcov-action@v1 - #- name: Coveralls - # uses: coverallsapp/github-action@master - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # path-to-lcov: coverage.lcov - #- name: Coverage Gate - # uses: VeryGoodOpenSource/very_good_coverage@v2 - # with: - # path: coverage.lcov - # min_coverage: 55 go-licenses: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 8c33040..25fa3d7 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,8 @@ pre-checks: go install honnef.co/go/tools/cmd/staticcheck@latest go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest -checks: fmt vet staticcheck golangci-lint test-cover +checks: + ./scripts/checks.sh fmt: go fmt ./... @@ -54,7 +55,7 @@ staticcheck: staticcheck ./... golangci-lint: - golangci-lint run + golangci-lint run --timeout 5m test-cover: ./scripts/test-cover.sh diff --git a/etc/.gitignore b/etc/.gitignore deleted file mode 100644 index a3e44f0..0000000 --- a/etc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -datasources.yaml \ No newline at end of file diff --git a/etc/datasources.example.yaml b/etc/datasources.yaml similarity index 57% rename from etc/datasources.example.yaml rename to etc/datasources.yaml index 58b3419..5e9112d 100644 --- a/etc/datasources.example.yaml +++ b/etc/datasources.yaml @@ -1,11 +1,7 @@ -# queryTimeOut: 30 datasources: - name: Prometheus type: prometheus url: http://prometheus:9090 - # basicAuth: true - # basicAuthUser: Aladdin - # basicAuthPassword: openSesame - name: Lethe type: lethe url: http://lethe:3100 diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go index 3b83402..b0a1569 100644 --- a/pkg/configuration/config.go +++ b/pkg/configuration/config.go @@ -12,7 +12,7 @@ import ( type Config struct { Version string - UserConfig *UsersConfig + UserConfig UsersConfig DatasourcesConfig *DatasourcesConfig //Dashboards []Dashboard //AlertRuleGroups []AlertRuleGroup @@ -73,8 +73,8 @@ func Load(version string) (*Config, error) { } defer userConfigFile.Close() - var userConf *UsersConfig - err = loadConfig(userConfigFile, userConf) + var userConf UsersConfig + err = loadConfig(userConfigFile, &userConf) if err != nil { return nil, fmt.Errorf("error on loading User Config: %w", err) } @@ -91,30 +91,6 @@ func Load(version string) (*Config, error) { return nil, fmt.Errorf("error on loading Datasources Config: %w", err) } - /* - dashboardfilepaths := glob("etc/dashboards", func(path string) bool { - return !strings.Contains(path, "/..") && filepath.Ext(path) == ".yaml" - }) - - for _, path := range dashboardfilepaths { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - var dashBoard *Dashboard - err = loadConfig(f, dashBoard) - if err != nil { - return nil, err - } - } - */ - - //loadDashboards() - //loadAlertRuleGroups() - - //datasourceStore = pkg.NewDatasourceStore(Config.DatasourcesConfig) - return &Config{ Version: version, UserConfig: userConf, diff --git a/pkg/configuration/config_test.go b/pkg/configuration/config_test.go index d8fafc8..fa3544b 100644 --- a/pkg/configuration/config_test.go +++ b/pkg/configuration/config_test.go @@ -2,11 +2,30 @@ package configuration import ( "bytes" - "github.com/stretchr/testify/assert" "io" + "os" "testing" + + "github.com/stretchr/testify/assert" ) +func init() { + _ = os.Chdir("../..") +} + +func TestLoad(t *testing.T) { + cfg, err := Load("Unknown") + assert.Nil(t, err) + assert.Equal(t, cfg.Version, "Unknown") + assert.Equal(t, cfg.UserConfig, UsersConfig{EtcUsers: []EtcUser{ + {Username: "admin", Hash: "$2a$12$VcCDgh2NDk07JGN0rjGbM.Ad41qVR/YFJcgHp0UGns5JDymv..TOG", IsAdmin: true}, + }}) + assert.ElementsMatch(t, cfg.DatasourcesConfig.Datasources, []*Datasource{ + {Type: DatasourceTypePrometheus, Name: "Prometheus", URL: "http://prometheus:9090", BasicAuth: false, BasicAuthUser: "", BasicAuthPassword: "", IsDefault: false, IsDiscovered: false}, + {Type: DatasourceTypeLethe, Name: "Lethe", URL: "http://lethe:3100", BasicAuth: false, BasicAuthUser: "", BasicAuthPassword: "", IsDefault: false, IsDiscovered: false}, + }) +} + func TestLoadDatasourcesConfig(t *testing.T) { tests := map[string]struct { diff --git a/pkg/handler/alert.go b/pkg/handler/alert.go index d65dd74..38579e4 100644 --- a/pkg/handler/alert.go +++ b/pkg/handler/alert.go @@ -13,6 +13,6 @@ func NewAlertHandler(r *store.AlertRuleStore) *alertHandler { return &alertHandler{r} } -func (ah *alertHandler) AlertRuleGroups(c *gin.Context) { - c.JSON(200, ah.AlertRuleStore.Groups()) +func (ah *alertHandler) AlertRuleGroupsList(c *gin.Context) { + c.JSON(200, ah.AlertRuleStore.RuleGroupsList()) } diff --git a/pkg/handler/dashboard.go b/pkg/handler/dashboard.go index da0fe11..4763b22 100644 --- a/pkg/handler/dashboard.go +++ b/pkg/handler/dashboard.go @@ -1,9 +1,10 @@ package handler import ( + "net/http" + "github.com/gin-gonic/gin" "github.com/kuoss/venti/pkg/store" - "net/http" ) // todo moved from config handler *should* modify web router path @@ -18,7 +19,7 @@ func NewDashboardHandler(ds *store.DashboardStore) *dashboardHandler { return &dashboardHandler{ds} } -//GET /dashboards +// GET /dashboards func (dh *dashboardHandler) Dashboards(c *gin.Context) { c.JSON(http.StatusOK, dh.DashboardStore.Dashboards()) } diff --git a/pkg/server.go b/pkg/server.go index 62cd764..9859000 100644 --- a/pkg/server.go +++ b/pkg/server.go @@ -2,6 +2,7 @@ package pkg import ( "fmt" + "github.com/gin-gonic/gin" "github.com/kuoss/venti/pkg/configuration" "github.com/kuoss/venti/pkg/handler" @@ -31,7 +32,7 @@ func LoadStores(config *configuration.Config) (*Stores, error) { return nil, fmt.Errorf("load datasource configuration failed: %w", err) } - userStore, err := store.NewUserService("./data/venti.sqlite3", config.UserConfig) + userStore, err := store.NewUserStore("./data/venti.sqlite3", config.UserConfig) if err != nil { return nil, fmt.Errorf("load user configuration failed: %w", err) } @@ -89,7 +90,7 @@ func LoadRouter(s *Stores, config *configuration.Config) *gin.Engine { alertGroup := api.Group("/alerts") { - alertGroup.GET("/", ah.AlertRuleGroups) + alertGroup.GET("/", ah.AlertRuleGroupsList) } letheGroup := api.Group("/lethe") diff --git a/pkg/store/alertRuleStore.go b/pkg/store/alertRuleStore.go new file mode 100644 index 0000000..677b0fd --- /dev/null +++ b/pkg/store/alertRuleStore.go @@ -0,0 +1,41 @@ +package store + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/prometheus/prometheus/model/rulefmt" +) + +type AlertRuleStore struct { + ruleGroupsList []rulefmt.RuleGroups +} + +func NewAlertRuleStore(pattern string) (*AlertRuleStore, error) { + log.Println("Loading alertRules...") + files, err := filepath.Glob("etc/alertrules/*.yaml") + if err != nil { + return nil, err + } + var ruleGroupsList []rulefmt.RuleGroups + for _, filename := range files { + log.Printf("alertRule file: %s\n", filename) + f, err := os.Open(filename) + if err != nil { + return nil, fmt.Errorf("error on Open: %w", err) + } + var ruleGroups *rulefmt.RuleGroups + err = loadYaml(f, &ruleGroups) + if err != nil { + return nil, fmt.Errorf("error on loadYaml: %w", err) + } + ruleGroupsList = append(ruleGroupsList, *ruleGroups) + } + return &AlertRuleStore{ruleGroupsList: ruleGroupsList}, nil +} + +func (ars *AlertRuleStore) RuleGroupsList() []rulefmt.RuleGroups { + return ars.ruleGroupsList +} diff --git a/pkg/store/alertRuleStore_test.go b/pkg/store/alertRuleStore_test.go new file mode 100644 index 0000000..2a46e8d --- /dev/null +++ b/pkg/store/alertRuleStore_test.go @@ -0,0 +1,26 @@ +package store + +import ( + "os" + "testing" + + "github.com/prometheus/prometheus/model/rulefmt" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func init() { + _ = os.Chdir("../..") +} + +func TestNewAlertRuleStore(t *testing.T) { + ars, err := NewAlertRuleStore("") + assert.Nil(t, err) + assert.Equal(t, ars, &AlertRuleStore{ruleGroupsList: []rulefmt.RuleGroups{rulefmt.RuleGroups{Groups: []rulefmt.RuleGroup{rulefmt.RuleGroup{Name: "sample", Interval: 0, Limit: 0, Rules: []rulefmt.RuleNode{rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S00-AlwaysOn", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 7, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "vector(1234)", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 8, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "AlwaysOn value={{ $value }}"}}, rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S01-Monday", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 12, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "day_of_week() == 1 and hour() < 2", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 13, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "Monday"}}, rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S02-NewNamespace", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 17, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "time() - kube_namespace_created < 120", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 18, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "labels={{ $labels }} namespace={{ $labels.namespace }} value={{ $value }}"}}}}}}}}) +} + +func TestRuleGroupsList(t *testing.T) { + ars, err := NewAlertRuleStore("") + assert.Nil(t, err) + assert.Equal(t, ars.RuleGroupsList(), []rulefmt.RuleGroups{rulefmt.RuleGroups{Groups: []rulefmt.RuleGroup{rulefmt.RuleGroup{Name: "sample", Interval: 0, Limit: 0, Rules: []rulefmt.RuleNode{rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S00-AlwaysOn", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 7, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "vector(1234)", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 8, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "AlwaysOn value={{ $value }}"}}, rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S01-Monday", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 12, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "day_of_week() == 1 and hour() < 2", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 13, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "Monday"}}, rulefmt.RuleNode{Record: yaml.Node{Kind: 0x0, Style: 0x0, Tag: "", Value: "", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 0, Column: 0}, Alert: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "S02-NewNamespace", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 17, Column: 12}, Expr: yaml.Node{Kind: 0x8, Style: 0x0, Tag: "!!str", Value: "time() - kube_namespace_created < 120", Anchor: "", Alias: (*yaml.Node)(nil), Content: []*yaml.Node(nil), HeadComment: "", LineComment: "", FootComment: "", Line: 18, Column: 11}, For: 0, Labels: map[string]string(nil), Annotations: map[string]string{"summary": "labels={{ $labels }} namespace={{ $labels.namespace }} value={{ $value }}"}}}}}}}) +} diff --git a/pkg/store/alertrulestore.go b/pkg/store/alertrulestore.go deleted file mode 100644 index 9065774..0000000 --- a/pkg/store/alertrulestore.go +++ /dev/null @@ -1,41 +0,0 @@ -package store - -import ( - "github.com/prometheus/prometheus/model/rulefmt" - "log" - "os" - "path/filepath" -) - -type AlertRuleStore struct { - ruleGroups rulefmt.RuleGroups -} - -func NewAlertRuleStore(pattern string) (*AlertRuleStore, error) { - log.Println("Loading alertrules...") - // default: "etc/alertrules/*.yaml" - files, err := filepath.Glob("etc/alertrules/*.yaml") - if err != nil { - return nil, err - } - - var alertRuleGroups rulefmt.RuleGroups - for _, filename := range files { - log.Printf("alertrulegroup file: %s\n", filename) - f, err := os.Open(filename) - if err != nil { - log.Printf("error open alertrulegroup file: %s\n", err.Error()) - } - var rg *rulefmt.RuleGroup - err = loadYaml(f, rg) - if err != nil { - return nil, err - } - alertRuleGroups.Groups = append(alertRuleGroups.Groups, *rg) - } - return &AlertRuleStore{ruleGroups: alertRuleGroups}, nil -} - -func (ars *AlertRuleStore) Groups() rulefmt.RuleGroups { - return ars.ruleGroups -} diff --git a/pkg/store/userStore.go b/pkg/store/userStore.go index 08ce86a..af352c0 100644 --- a/pkg/store/userStore.go +++ b/pkg/store/userStore.go @@ -2,9 +2,10 @@ package store import ( "fmt" + "log" + "github.com/kuoss/venti/pkg/auth" "github.com/kuoss/venti/pkg/configuration" - "log" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -17,7 +18,7 @@ type UserStore struct { db *gorm.DB } -func NewUserService(filepath string, config *configuration.UsersConfig) (*UserStore, error) { +func NewUserStore(filepath string, config configuration.UsersConfig) (*UserStore, error) { log.Println("Initializing database...") db, err := gorm.Open(sqlite.Open(filepath), &gorm.Config{}) @@ -33,7 +34,7 @@ func NewUserService(filepath string, config *configuration.UsersConfig) (*UserSt return &UserStore{db}, nil } -func setEtcUsers(db *gorm.DB, config *configuration.UsersConfig) { +func setEtcUsers(db *gorm.DB, config configuration.UsersConfig) { for _, etcUser := range config.EtcUsers { var user auth.User diff --git a/scripts/checks.sh b/scripts/checks.sh new file mode 100755 index 0000000..2099547 --- /dev/null +++ b/scripts/checks.sh @@ -0,0 +1,9 @@ +#!/bin/bash +cd $(dirname $0)/.. + +set -xeuo pipefail +go fmt ./... +go vet ./... +staticcheck ./... +golangci-lint run --timeout 5m +./scripts/test-cover.sh diff --git a/scripts/test-cover.sh b/scripts/test-cover.sh index 3b9d694..a9cecb2 100755 --- a/scripts/test-cover.sh +++ b/scripts/test-cover.sh @@ -2,7 +2,7 @@ MIN_COVER=50.0 cd $(dirname $0)/.. -go test ./... -failfast -coverprofile cover.out +go test ./... -failfast -race -coverprofile cover.out if [[ $? != 0 ]]; then echo "❌ FAIL ( test failed )" exit 1