feat: Add prometheus timeout (#2893)

* feat: add prometheus timeout attribute Signed-off-by: AhmedGrati <[email protected]> * add protobuf and crd of timeout attribute Signed-off-by: AhmedGrati <[email protected]> * docs: add timeout attribute in the prometheus docs Signed-off-by: AhmedGrati <[email protected]> * make private Signed-off-by: zachaller <[email protected]> --------- Signed-off-by: AhmedGrati <[email protected]> Signed-off-by: zachaller <[email protected]> Co-authored-by: zachaller <[email protected]>
argoproj · Jul 25, 2023 · d8dbb24 · d8dbb24
1 parent 9365aca
commit d8dbb24
Show file tree

Hide file tree

Showing 14 changed files with 389 additions and 245 deletions.
diff --git a/docs/analysis/prometheus.md b/docs/analysis/prometheus.md
@@ -20,6 +20,8 @@ spec:
     provider:
       prometheus:
         address: http://prometheus.example.com:9090
+        # timeout is expressed in seconds
+        timeout: 40
         query: |
           sum(irate(
             istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}",response_code!~"5.*"}[5m]

diff --git a/docs/features/kustomize/rollout_cr_schema.json b/docs/features/kustomize/rollout_cr_schema.json
@@ -4280,6 +4280,10 @@
                                                     },
                                                     "query": {
                                                         "type": "string"
+                                                    },
+                                                    "timeout": {
+                                                        "format": "int64",
+                                                        "type": "integer"
                                                     }
                                                 },
                                                 "type": "object"
@@ -8670,6 +8674,10 @@
                                                     },
                                                     "query": {
                                                         "type": "string"
+                                                    },
+                                                    "timeout": {
+                                                        "format": "int64",
+                                                        "type": "integer"
                                                     }
                                                 },
                                                 "type": "object"
@@ -13060,6 +13068,10 @@
                                                     },
                                                     "query": {
                                                         "type": "string"
+                                                    },
+                                                    "timeout": {
+                                                        "format": "int64",
+                                                        "type": "integer"
                                                     }
                                                 },
                                                 "type": "object"

diff --git a/manifests/crds/analysis-run-crd.yaml b/manifests/crds/analysis-run-crd.yaml
@@ -2779,6 +2779,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:

diff --git a/manifests/crds/analysis-template-crd.yaml b/manifests/crds/analysis-template-crd.yaml
@@ -2775,6 +2775,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:

diff --git a/manifests/crds/cluster-analysis-template-crd.yaml b/manifests/crds/cluster-analysis-template-crd.yaml
@@ -2775,6 +2775,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:

diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -2780,6 +2780,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:
@@ -5739,6 +5742,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:
@@ -8584,6 +8590,9 @@ spec:
                               type: object
                             query:
                               type: string
+                            timeout:
+                              format: int64
+                              type: integer
                           type: object
                         skywalking:
                           properties:

diff --git a/metricproviders/metricproviders.go b/metricproviders/metricproviders.go
@@ -41,7 +41,7 @@ func (f *ProviderFactory) NewProvider(logCtx log.Entry, metric v1alpha1.Metric)
 		if err != nil {
 			return nil, err
 		}
-		return prometheus.NewPrometheusProvider(api, logCtx), nil
+		return prometheus.NewPrometheusProvider(api, logCtx, metric)
 	case job.ProviderType:
 		return job.NewJobProvider(logCtx, f.KubeClient, f.JobLister), nil
 	case kayenta.ProviderType:

diff --git a/metricproviders/prometheus/prometheus.go b/metricproviders/prometheus/prometheus.go
@@ -33,8 +33,9 @@ const (
 
 // Provider contains all the required components to run a prometheus query
 type Provider struct {
-	api    v1.API
-	logCtx log.Entry
+	api     v1.API
+	logCtx  log.Entry
+	timeout time.Duration
 }
 
 // Type indicates provider is a prometheus provider
@@ -59,7 +60,7 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph
 	}
 
 	//TODO(dthomson) make timeout configurable
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), p.timeout)
 	defer cancel()
 
 	response, warnings, err := p.api.Query(ctx, metric.Provider.Prometheus.Query, time.Now())
@@ -138,11 +139,25 @@ func (p *Provider) processResponse(metric v1alpha1.Metric, response model.Value)
 }
 
 // NewPrometheusProvider Creates a new Prometheus client
-func NewPrometheusProvider(api v1.API, logCtx log.Entry) *Provider {
-	return &Provider{
+func NewPrometheusProvider(api v1.API, logCtx log.Entry, metric v1alpha1.Metric) (*Provider, error) {
+	provider := &Provider{
 		logCtx: logCtx,
 		api:    api,
 	}
+
+	if metric.Provider.Prometheus == nil || metric.Provider.Prometheus.Timeout == nil {
+		provider.timeout = 30 * time.Second
+		return provider, nil
+	}
+
+	metricTimeout := metric.Provider.Prometheus.Timeout
+
+	if *metricTimeout < 0 {
+		return nil, errors.New("prometheus timeout should not be negative")
+	}
+
+	provider.timeout = time.Duration(*metricTimeout * int64(time.Second))
+	return provider, nil
 }
 
 // NewPrometheusAPI generates a prometheus API from the metric configuration