Monitoring: support to change file persistence

cmd/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/hashicorp/go-version v1.2.0
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pkg/errors v0.8.1
+	github.com/prometheus/common v0.4.1
 	github.com/prometheus/prometheus v0.0.0-20190710134608-e5b22494857d
 	github.com/rakyll/statik v0.1.6
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749

monitor/v2.1.10/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.11/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.12/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.13/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.14/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.15/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.16/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.8/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v2.1.9/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0-beta.1/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0-beta/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0-rc.1/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0-rc.2/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0-rc.3/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.0/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.1/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.2/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

monitor/v3.0.3/init.sh

@@ -50,6 +50,14 @@ for file in $PROM_CONFIG_PATH/rules/*
 do
     sed -i 's/ENV_LABELS_ENV/'$TIDB_CLUSTER_NAME'/g' $file
 done
+# Copy Persistent rules to override raw files
+if [ $PROM_PERSISTENT_DIR ];
+then
+    if [ -d $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/} ];then
+        cp -f $PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}/*.rules.yml $PROM_CONFIG_PATH/rules
+    fi
+fi
+
 
 # Datasources
 if [ $GF_DATASOURCE_PATH ];

reload/main.go

@@ -2,6 +2,7 @@
 package main
 
 import (
+	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/pingcap/monitoring/reload/server/apis"
 	"github.com/pingcap/monitoring/reload/server/bizlogic"
@@ -12,11 +13,15 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
+	"path/filepath"
 )
 
 var (
 	watchDir        string
 	promUrl         string
+	rootStorePath 	string
+	subStorePath	string
 )
 
 func main() {
@@ -25,11 +30,17 @@ func main() {
 		Run: func(co *cobra.Command, args []string) {
 			startServer()
 		},
+		FParseErrWhitelist: cobra.FParseErrWhitelist{
+			UnknownFlags: true,
+		},
 	}
 
 	rootCmd.Flags().StringVar(&watchDir,"watch-path", "", "the directory to watch for updates.")
+	rootCmd.Flags().StringVar(&rootStorePath,"root-store-path", "/data", "the directory to store updated files.")
+	rootCmd.Flags().StringVar(&subStorePath,"sub-store-path", "", "the sub directory to store updated files.")
 	rootCmd.Flags().StringVar(&promUrl,"prometheus-url", "http://localhost:9090", "the url to send a request to when the files are updated.")
 	rootCmd.MarkFlagRequired("watch-path")
+
 	rootCmd.Execute()
 }
 
@@ -39,7 +50,15 @@ func startServer() {
 		panic(errors.Wrap(err, "invalid url"))
 	}
 
-	engine := apis.NewService(u, watchDir)
+	storePath := fmt.Sprintf("%s%clatest-rules%c%s", rootStorePath, filepath.Separator, filepath.Separator, extract(subStorePath))
+	if len(subStorePath) != 0 && !exist(storePath) {
+		log.Println("need to store latest file to store path")
+		if err := os.MkdirAll(storePath, os.ModePerm); err != nil {
+			log.Println("failed to create store path", err)
+		}
+	}
+
+	engine := apis.NewService(u, watchDir, len(subStorePath) != 0, storePath)
 	statikFS, err := fs.New()
 	if err != nil {
 		log.Fatal("failed to embed static files into your binary", err)
@@ -55,4 +74,21 @@ func startServer() {
 	}
 
 	log.Fatal("StartServer server failed", engine.Run("0.0.0.0:9089").Error())
+}
+
+func extract(path string) string {
+	for i := len(path) - 1; i >= 0; i-- {
+		if path[i] == filepath.Separator {
+			return path[i + 1:]
+		}
+	}
+	return path
+}
+
+func exist(filepath string) bool{
+	if _, err := os.Stat(filepath); err != nil {
+		return false
+	}
+
+	return true
 }

reload/server/apis/server.go

@@ -6,8 +6,8 @@ import (
 	"net/url"
 )
 
-func NewService(promUrl *url.URL, watchDir string) *gin.Engine {
-	s := bizlogic.NewServer(promUrl, watchDir)
+func NewService(promUrl *url.URL, watchDir string, needStoreFileToStorePath bool, storePath string) *gin.Engine {
+	s := bizlogic.NewServer(promUrl, watchDir, needStoreFileToStorePath, storePath)
 	r := gin.Default()
 
 	ws := r.Group("monitoring")