Skip to content

Commit

Permalink
cluster: fix Grafana error datasource not found (#1768)
Browse files Browse the repository at this point in the history
  • Loading branch information
srstack authored and AstroProfundis committed Feb 23, 2022
1 parent 92b4bd1 commit 3addfc2
Show file tree
Hide file tree
Showing 5 changed files with 24 additions and 16 deletions.
4 changes: 1 addition & 3 deletions components/playground/grafana.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,6 @@ func writeDatasourceConfig(fname string, clusterName string, p8sURL string) erro
}

tpl := `apiVersion: 1
deleteDatasources:
- name: %s
datasources:
- name: %s
type: prometheus
Expand All @@ -70,7 +68,7 @@ datasources:
editable: true
`

s := fmt.Sprintf(tpl, clusterName, clusterName, p8sURL)
s := fmt.Sprintf(tpl, clusterName, p8sURL)
err = os.WriteFile(fname, []byte(s), 0644)
if err != nil {
return errors.AddStack(err)
Expand Down
2 changes: 0 additions & 2 deletions embed/templates/config/datasource.yml.tpl
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
apiVersion: 1
deleteDatasources:
- name: {{.ClusterName}}
datasources:
- name: {{.ClusterName}}
type: prometheus
Expand Down
28 changes: 19 additions & 9 deletions pkg/cluster/manager/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,29 +32,38 @@ import (
"github.com/pingcap/tiup/pkg/utils"
)

// buildReloadPromTasks reloads Prometheus configuration
func buildReloadPromTasks(
// buildReloadPromTasks reloads Prometheus and Grafana configuration
func buildReloadPromAndGrafanaTasks(
topo spec.Topology,
logger *logprinter.Logger,
gOpt operator.Options,
nodes ...string,
) []*task.StepDisplay {
var instances []spec.Instance
// get promtheus and grafana instance list
monitor := spec.FindComponent(topo, spec.ComponentPrometheus)
if monitor == nil {
return nil
}
instances := monitor.Instances()
grafanas := spec.FindComponent(topo, spec.ComponentGrafana)

instances = append(instances, monitor.Instances()...)
instances = append(instances, grafanas.Instances()...)

if len(instances) == 0 {
return nil
}
var tasks []*task.StepDisplay
deletedNodes := set.NewStringSet(nodes...)
for _, inst := range monitor.Instances() {
for _, inst := range instances {
if deletedNodes.Exist(inst.ID()) {
continue
}
// reload Prometheus
action := "reload"
if inst.ComponentName() == spec.ComponentGrafana {
// restart grafana
action = "restart"
}
t := task.NewBuilder(logger).
SystemCtl(inst.GetHost(), inst.ServiceName(), "reload", true).
SystemCtl(inst.GetHost(), inst.ServiceName(), action, true).
BuildAsStep(fmt.Sprintf(" - Reload %s -> %s", inst.ComponentName(), inst.ID()))
tasks = append(tasks, t)
}
Expand Down Expand Up @@ -341,7 +350,8 @@ func buildScaleOutTask(
return operator.Start(ctx, newPart, operator.Options{OptTimeout: gOpt.OptTimeout, Operation: operator.ScaleOutOperation}, tlsCfg)
}).
ParallelStep("+ Refresh components conifgs", gOpt.Force, refreshConfigTasks...).
ParallelStep("+ Reload prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...)
ParallelStep("+ Reload prometheus and grafana", gOpt.Force,
buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt)...)
}

// remove scale-out file lock
Expand Down
3 changes: 2 additions & 1 deletion pkg/cluster/manager/destroy.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ func (m *Manager) DestroyTombstone(
UpdateMeta(name, clusterMeta, nodes).
UpdateTopology(name, m.specManager.Path(name), clusterMeta, nodes).
ParallelStep("+ Refresh instance configs", gOpt.Force, regenConfigTasks...).
ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt)...).
ParallelStep("+ Reloda prometheus and grafana", gOpt.Force,
buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt)...).
Build()

if err := t.Execute(ctx); err != nil {
Expand Down
3 changes: 2 additions & 1 deletion pkg/cluster/manager/scale_in.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@ func (m *Manager) ScaleIn(

t := b.
ParallelStep("+ Refresh instance configs", force, regenConfigTasks...).
ParallelStep("+ Reloda prometheus", gOpt.Force, buildReloadPromTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...).
ParallelStep("+ Reloda prometheus and grafana", gOpt.Force,
buildReloadPromAndGrafanaTasks(metadata.GetTopology(), m.logger, gOpt, nodes...)...).
Build()

ctx := ctxt.New(
Expand Down

0 comments on commit 3addfc2

Please sign in to comment.