Mentix PromSD extensions (#2560)

changelog/unreleased/mentix-prom-ext.md

@@ -0,0 +1,5 @@
+Enhancement: Mentix PromSD extensions
+
+The Mentix Prometheus SD scrape targets are now split into one file per service type, making health checks configuration easier. Furthermore, the local file connector for mesh data and the site registration endpoint have been dropped, as they aren't needed anymore.
+
+https://github.com/cs3org/reva/pull/2560

docs/content/en/docs/config/http/services/mentix/_index.md

@@ -43,17 +43,13 @@ _Supported connectors:_
 
 - **gocdb** 
 The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)).
-
-- **localfile**
-The [localfile](localfile) connector reads sites from a local JSON file. The file must contain an array of sites adhering to the `meshdata.Site` structure.
 
 ## Importers
 Mentix can import mesh data from various sources and write it to one or more targets through the corresponding connectors.
 
 __Supported importers:__
 
-- **sitereg**
-Mentix can import new sites via an HTTP endpoint using the `sitereg` importer. Data can be sent to the configured relative endpoint (see [here](sitereg)).
+- **None**
 
 ## Exporters
 Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint.

docs/content/en/docs/config/http/services/mentix/promsd/_index.md

@@ -10,19 +10,11 @@ description: >
 When using the Prometheus SD exporter, the output filenames have to be configured first.
 {{% /pageinfo %}}
 
-{{% dir name="metrics_output_file" type="string" default="" %}}
-The target filename of the generated Prometheus File SD scrape config for metrics.
+{{% dir name="output_path" type="string" default="" %}}
+The target path of the generated Prometheus File SD scrape configs for metrics.
 {{< highlight toml >}}
 [http.services.mentix.exporters.promsd]
-metrics_output_file = "/var/shared/prometheus/sciencemesh.json"
-{{< /highlight >}}
-{{% /dir %}}
-
-{{% dir name="blackbox_output_file" type="string" default="" %}}
-The target filename of the generated Prometheus File SD scrape config for the blackbox exporter.
-{{< highlight toml >}}
-[http.services.mentix.exporters.promsd]
-blackbox_output_file = "/var/shared/prometheus/blackbox.json"
+output_path = "/var/shared/prometheus/sciencemesh"
 {{< /highlight >}}
 {{% /dir %}}
 

examples/mentix/mentix.toml

@@ -8,10 +8,6 @@ update_interval = "15m"
 address = "http://sciencemesh-test.uni-muenster.de"
 apikey = "abc123"
 
-# Sites can also be stored in a local file
-[http.services.mentix.connectors.localfile]
-file = "/usr/share/revad/sites.json"
-
 # Configure the service types that are considered as critical/essential
 [http.services.mentix.services]
 critical_types = ["REVAD"]
@@ -28,13 +24,6 @@ enabled_connectors = ["gocdb"]
 # Enable the Metrics exporter
 [http.services.mentix.exporters.metrics]
 
-# Enable the site registration importer
-[http.services.mentix.importers.sitereg]
-# For importers, this is obligatory; the connectors will be used as the target for data updates
-enabled_connectors = ["localfile"]
-# If set to true, ScienceMesh sites will be ignored when they try to register
-ignore_sm_sites = false
-
 # Set up the accounts service used to query information about accounts associated with registered sites
 [http.services.mentix.accounts]
 # Depending on where the service is running, localhost may also be used here
@@ -44,7 +33,6 @@ password = "userpass"
 
 # Configure the Prometheus Service Discovery:
 [http.services.mentix.exporters.promsd]
-# The following files must be made available to Prometheus.
-# They can then be used as the file_sd source of a job.
-metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json"
-blackbox_output_file = "/usr/share/prom/sciencemesh_blackbox.json"
+# The following path must be made available to Prometheus.
+# The contained target files can then be used as the file_sd sources of a job.
+output_path = "/usr/share/prom"

internal/http/services/mentix/mentix.go

@@ -137,11 +137,6 @@ func applyDefaultConfig(conf *config.Configuration) {
 		conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future
 	}
 
-	// Importers
-	if conf.Importers.SiteRegistration.Endpoint == "" {
-		conf.Importers.SiteRegistration.Endpoint = "/sitereg"
-	}
-
 	// Exporters
 	addDefaultConnector := func(enabledList *[]string) {
 		if len(*enabledList) == 0 {

pkg/mentix/config/config.go

@@ -28,10 +28,6 @@ type Configuration struct {
 			Scope   string `mapstructure:"scope"`
 			APIKey  string `mapstructure:"apikey"`
 		} `mapstructure:"gocdb"`
-
-		LocalFile struct {
-			File string `mapstructure:"file"`
-		} `mapstructure:"localfile"`
 	} `mapstructure:"connectors"`
 
 	UpdateInterval string `mapstructure:"update_interval"`
@@ -40,15 +36,6 @@ type Configuration struct {
 		CriticalTypes []string `mapstructure:"critical_types"`
 	} `mapstructure:"services"`
 
-	Importers struct {
-		SiteRegistration struct {
-			Endpoint               string   `mapstructure:"endpoint"`
-			EnabledConnectors      []string `mapstructure:"enabled_connectors"`
-			IsProtected            bool     `mapstructure:"is_protected"`
-			IgnoreScienceMeshSites bool     `mapstructure:"ignore_sm_sites"`
-		} `mapstructure:"sitereg"`
-	} `mapstructure:"importers"`
-
 	Exporters struct {
 		WebAPI struct {
 			Endpoint          string   `mapstructure:"endpoint"`
@@ -70,9 +57,8 @@ type Configuration struct {
 		} `mapstructure:"siteloc"`
 
 		PrometheusSD struct {
-			MetricsOutputFile  string   `mapstructure:"metrics_output_file"`
-			BlackboxOutputFile string   `mapstructure:"blackbox_output_file"`
-			EnabledConnectors  []string `mapstructure:"enabled_connectors"`
+			OutputPath        string   `mapstructure:"output_path"`
+			EnabledConnectors []string `mapstructure:"enabled_connectors"`
 		} `mapstructure:"promsd"`
 
 		Metrics struct {

pkg/mentix/exchangers/exporters/promsd.go

@@ -24,7 +24,9 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
+	"strings"
 
 	"github.com/cs3org/reva/pkg/mentix/utils"
 	"github.com/rs/zerolog"
@@ -54,6 +56,7 @@ const (
 	labelSiteID      = "__meta_mentix_site_id"
 	labelSiteCountry = "__meta_mentix_site_country"
 	labelType        = "__meta_mentix_type"
+	labelURL         = "__meta_mentix_url"
 	labelScheme      = "__meta_mentix_scheme"
 	labelHost        = "__meta_mentix_host"
 	labelPort        = "__meta_mentix_port"
@@ -78,6 +81,7 @@ func getScrapeTargetLabels(site *meshdata.Site, service *meshdata.Service, endpo
 		labelSiteID:      site.ID,
 		labelSiteCountry: site.CountryCode,
 		labelType:        endpoint.Type.Name,
+		labelURL:         endpoint.URL,
 		labelScheme:      endpointURL.Scheme,
 		labelHost:        endpointURL.Hostname(),
 		labelPort:        endpointURL.Port(),
@@ -110,12 +114,13 @@ func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Config
 	}
 
 	// Register all scrape creators
-	if err := registerCreator("metrics", conf.Exporters.PrometheusSD.MetricsOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointMetrics}); err != nil {
-		return fmt.Errorf("unable to register the 'metrics' scrape config creator: %v", err)
-	}
+	for _, endpoint := range meshdata.GetServiceEndpoints() {
+		epName := strings.ToLower(endpoint)
+		filename := path.Join(conf.Exporters.PrometheusSD.OutputPath, "svc_"+epName+".json")
 
-	if err := registerCreator("blackbox", conf.Exporters.PrometheusSD.BlackboxOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointGateway}); err != nil {
-		return fmt.Errorf("unable to register the 'blackbox' scrape config creator: %v", err)
+		if err := registerCreator(epName, filename, createGenericScrapeConfig, []string{endpoint}); err != nil {
+			return fmt.Errorf("unable to register the '%v' scrape config creator: %v", epName, err)
+		}
 	}
 
 	return nil
@@ -199,6 +204,10 @@ func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback promet
 		}
 	}
 
+	if scrapes == nil {
+		scrapes = []*prometheus.ScrapeConfig{}
+	}
+
 	return scrapes
 }