diff --git a/chart/stash/templates/deployment.yaml b/chart/stash/templates/deployment.yaml index ebc679cde..d887bd5a8 100644 --- a/chart/stash/templates/deployment.yaml +++ b/chart/stash/templates/deployment.yaml @@ -45,7 +45,6 @@ spec: - --enable-analytics={{ .Values.enableAnalytics }} ports: - containerPort: 8443 - - containerPort: 56790 readinessProbe: httpGet: path: /healthz diff --git a/chart/stash/templates/service.yaml b/chart/stash/templates/service.yaml index 9c2c6a3a4..90d03e6e1 100644 --- a/chart/stash/templates/service.yaml +++ b/chart/stash/templates/service.yaml @@ -19,11 +19,6 @@ spec: port: 56789 protocol: TCP targetPort: 56789 - # Port used to expose Prometheus metrics for the operator - - name: ops - port: 56790 - protocol: TCP - targetPort: 56790 selector: app: "{{ template "stash.name" . }}" release: "{{ .Release.Name }}" diff --git a/docs/examples/monitoring/metrics-collector.yaml b/docs/examples/monitoring/metrics-collector.yaml new file mode 100644 index 000000000..e4377de77 --- /dev/null +++ b/docs/examples/monitoring/metrics-collector.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: appscode:system:metrics-collector +rules: +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: appscode:system:metrics-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: appscode:system:metrics-collector +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous diff --git a/docs/examples/monitoring/profiler.yaml b/docs/examples/monitoring/profiler.yaml new file mode 100644 index 000000000..1e1a35030 --- /dev/null +++ b/docs/examples/monitoring/profiler.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: appscode:system:profiler +rules: +- nonResourceURLs: ["/debug/pprof/", "/debug/pprof/*"] + verbs: ["get", "post"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: appscode:system:profiler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: appscode:system:profiler +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous diff --git a/docs/guides/monitoring.md b/docs/guides/monitoring.md index a384ce279..5ec96406f 100644 --- a/docs/guides/monitoring.md +++ b/docs/guides/monitoring.md @@ -19,7 +19,64 @@ section_menu_id: guides Stash has native support for monitoring via Prometheus. ## Monitoring Stash Operator -Stash operator exposes Prometheus native monitoring data via `/metrics` endpoint on `:56790` port. You can setup a [CoreOS Prometheus ServiceMonitor](https://github.com/coreos/prometheus-operator) using `stash-operator` service. +Stash operator exposes Prometheus native monitoring data via `/metrics` endpoint on `:8443` port. You can setup a [CoreOS Prometheus ServiceMonitor](https://github.com/coreos/prometheus-operator) using `stash-operator` service. + + +Follow the steps below to view the metrics: + +1. Give `system:anonymous` user access to `/metrics` url. **This is not safe to do on a production cluster.** + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: appscode:system:metrics-collector +rules: +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: appscode:system:metrics-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: appscode:system:metrics-collector +subjects: +- apiGroup: rbac.authorization.k8s.io + kind: User + name: system:anonymous +``` + +```console +$ kubectl auth reconcile -f docs/examples/monitoring/metrics-collector.yaml +clusterrole.rbac.authorization.k8s.io "appscode:system:metrics-collector" reconciled +clusterrolebinding.rbac.authorization.k8s.io "appscode:system:metrics-collector" reconciled +``` + +2. Now, forward the port `:8443` to your workstation. + +``` +$ kubectl get pods -n kube-system | grep stash +stash-operator-f89dcccdb-plvmt 1/1 Running 0 27m + +$ kubectl port-forward -n kube-system stash-operator-f89dcccdb-plvmt 8443 +Forwarding from 127.0.0.1:8443 -> 8443 +Forwarding from [::1]:8443 -> 8443 +``` + +3. Now, visit the url: https://127.0.0.1:8443/metrics + +![operator-metrics](/docs/images/monitoring/operator-metrics.png) + +4. Once you are done, remove access to `system:anonymous` user. + +```console +$ kubectl delete -f docs/examples/monitoring/metrics-collector.yaml +clusterrole.rbac.authorization.k8s.io "appscode:system:metrics-collector" deleted +clusterrolebinding.rbac.authorization.k8s.io "appscode:system:metrics-collector" deleted +``` ## Monitoring Backup Operation Since backup operations are run as cron jobs, Stash can use [Prometheus Pushgateway](https://github.com/prometheus/pushgateway) cache metrics for backup operation. The installation scripts for Stash operator deploys a Prometheus Pushgateway as a sidecar container. You can configure a Prometheus server to scrape this Pushgateway via `stash-operator` service on port `:56789`. Backup operations send the following metrics to this Pushgateway: diff --git a/docs/images/monitoring/operator-metrics.png b/docs/images/monitoring/operator-metrics.png new file mode 100644 index 000000000..1f6fc1e02 Binary files /dev/null and b/docs/images/monitoring/operator-metrics.png differ diff --git a/docs/images/monitoring/operator-profiler.png b/docs/images/monitoring/operator-profiler.png new file mode 100644 index 000000000..c047fd209 Binary files /dev/null and b/docs/images/monitoring/operator-profiler.png differ diff --git a/docs/reference/stash_run.md b/docs/reference/stash_run.md index c67fadde8..c02750f3a 100644 --- a/docs/reference/stash_run.md +++ b/docs/reference/stash_run.md @@ -63,7 +63,6 @@ stash run [flags] --http2-max-streams-per-connection int The limit that the server gives to clients for the maximum number of streams in an HTTP/2 connection. Zero means to use golang's default. (default 1000) --image-tag string Image tag for sidecar, init-container, check-job and recovery-job (default "canary") --kubeconfig string kubeconfig file pointing at the 'core' kubernetes server. - --ops-address string Address to listen on for web interface and telemetry. (default ":56790") --profiling Enable profiling via web interface host:port/debug/pprof/ (default true) --qps float The maximum QPS to the master from this client (default 100) --rbac Enable RBAC for operator diff --git a/hack/deploy/operator.yaml b/hack/deploy/operator.yaml index fdb30d9c4..c47fde6da 100644 --- a/hack/deploy/operator.yaml +++ b/hack/deploy/operator.yaml @@ -35,7 +35,6 @@ spec: - --enable-analytics=${STASH_ENABLE_ANALYTICS} ports: - containerPort: 8443 - - containerPort: 56790 resources: requests: cpu: "100m" @@ -103,8 +102,5 @@ spec: - name: pushgateway port: 56789 targetPort: 56789 - - name: ops - port: 56790 - targetPort: 56790 selector: app: stash diff --git a/hack/docker/setup.sh b/hack/docker/setup.sh index f73e21d6d..ffdda931a 100755 --- a/hack/docker/setup.sh +++ b/hack/docker/setup.sh @@ -81,7 +81,7 @@ COPY restic /bin/restic COPY stash /bin/stash ENTRYPOINT ["/bin/stash"] -EXPOSE 56789 56790 +EXPOSE 56789 EOL local cmd="docker build -t $DOCKER_REGISTRY/$IMG:$TAG ." echo $cmd; $cmd