Skip to content

Commit

Permalink
Replace haproxy-exporter by embedded one (#2777)
Browse files Browse the repository at this point in the history
* Enable embedded prometheus-exporter

* Scrape HAproxy metrics in Prometheus

* Remove haproxy-exporter

* Spec tests update

* Docs update

* Remove haproxy_exporter if upgrade

* Changes after review: remove haproxy-exporter if upgrade, add separate haproxy listener for metrics, add separate port for stats, add docs

* Change HAProxy dashboard

* Remove haproxy-exporter target, update Prometheus config during the upgrade
rafzei authored Jan 4, 2022
1 parent 134ba13 commit 12b99bf
Showing 40 changed files with 238 additions and 335 deletions.
11 changes: 0 additions & 11 deletions ansible/playbooks/haproxy_exporter.yml

This file was deleted.

2 changes: 1 addition & 1 deletion ansible/playbooks/prometheus.yml
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@
# Ansible playbook that makes sure the base items for all nodes are installed

- hosts: all
gather_facts: yes
gather_facts: true
tasks: [ ]

- hosts: prometheus
8 changes: 7 additions & 1 deletion ansible/playbooks/roles/haproxy/templates/haproxy.cfg.j2
Original file line number Diff line number Diff line change
@@ -92,11 +92,17 @@ backend {{ back.name }}

{%- if specification.stats is defined and specification.stats.enable == True %}
listen stats
bind {{ specification.stats.bind_address }}
bind {{ specification.stats.bind_address }}:{{ specification.stats.port }}
stats enable
stats refresh 10s
stats admin if { src 127.0.0.1 }
stats hide-version # Hide HAProxy version
stats uri {{ specification.stats.uri }}
stats auth {{ specification.stats.user }}:{{ specification.stats.password }}
{%- endif %}

{%- if specification.metrics is defined and specification.metrics.enable == True %}
listen metrics
bind {{ specification.metrics.bind_address }}:{{ specification.metrics.port }}
http-request use-service prometheus-exporter if { path /metrics }
{%- endif %}
4 changes: 0 additions & 4 deletions ansible/playbooks/roles/haproxy_exporter/defaults/main.yml

This file was deleted.

3 changes: 0 additions & 3 deletions ansible/playbooks/roles/haproxy_exporter/meta/main.yml

This file was deleted.

120 changes: 0 additions & 120 deletions ansible/playbooks/roles/haproxy_exporter/tasks/main.yml

This file was deleted.

This file was deleted.

This file was deleted.

2 changes: 0 additions & 2 deletions ansible/playbooks/roles/preflight/defaults/main.yml
Original file line number Diff line number Diff line change
@@ -53,7 +53,6 @@ unsupported_roles:
- kafka_exporter
- postgresql
- postgres_exporter
- haproxy_exporter
- applications
- ignite
- distro: CentOS
@@ -94,7 +93,6 @@ unsupported_roles:
- kafka_exporter
- postgresql
- postgres_exporter
- haproxy_exporter
- applications
- ignite

37 changes: 29 additions & 8 deletions ansible/playbooks/roles/prometheus/tasks/configure.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,36 @@
---
- name: Load RabbitMQ's config vars
include_vars:
file: roles/rabbitmq/vars/main.yml
name: rabbitmq_config
- name: Check if Alertmanager is enabled
set_fact:
is_alertmanager_enabled: "{{ specification.alertmanager.enable | bool and (specification.alertmanager.config.receivers | length>0 ) }}"

- name: Check RabbitMQ
when:
- groups.rabbitmq is defined
- groups.rabbitmq | length > 0
block:
- name: Load RabbitMQ's config vars
include_vars:
file: roles/rabbitmq/vars/main.yml
name: rabbitmq_config

- name: Set fact about RabbitMQ monitoring
set_fact:
rabbitmq_monitoring_enabled: "{{ rabbitmq_config.specification.rabbitmq_monitoring_enabled }}"
when: rabbitmq_config is defined
- name: Set fact about RabbitMQ monitoring
set_fact:
rabbitmq_monitoring_enabled: "{{ rabbitmq_config.specification.rabbitmq_monitoring_enabled }}"

- name: Check HAproxy
when:
- groups.haproxy is defined
- groups.haproxy | length > 0
block:
- name: Load HAproxy's config vars
include_vars:
file: roles/haproxy/vars/main.yml
name: haproxy_config

- name: Set fact about HAproxy monitoring
set_fact:
haproxy_monitoring_enabled: "{{ haproxy_config.specification.metrics.enable }}"
haproxy_monitoring_port: "{{ haproxy_config.specification.metrics.port }}"

- name: Configure prometheus
template:
@@ -21,6 +41,7 @@
group: prometheus
mode: u=rw,g=r,o=
validate: "/usr/local/bin/promtool check config %s"
backup: true
notify:
- Restart Prometheus

3 changes: 0 additions & 3 deletions ansible/playbooks/roles/prometheus/tasks/install.yml
Original file line number Diff line number Diff line change
@@ -37,9 +37,6 @@
set_fact:
binary_file_name: "{{ prometheus.file_name[ansible_architecture] }}"

- name: Package
debug: msg="{{ binary_file_name }}"

- name: Download Prometheus binaries
include_role:
name: download
8 changes: 2 additions & 6 deletions ansible/playbooks/roles/prometheus/tasks/main.yml
Original file line number Diff line number Diff line change
@@ -22,10 +22,6 @@
- "{{ ansible_distribution | lower }}.yml"
- "{{ ansible_os_family | lower }}.yml"

- name: "Check if Alertmanager is enabled"
set_fact:
is_alertmanager_enabled: "{{ specification.alertmanager.enable | bool and (specification.alertmanager.config.receivers | length>0 ) }}"

- name: Include install
include_tasks: install.yml

@@ -34,12 +30,12 @@

- name: Include tasks if Alertmanager is enabled
block:
- name: Include provide-rules
- name: Include provide-rules
include_tasks: provide-rules.yml

- name: Include install-alertmanager
include_tasks: install-alertmanager.yml

- name: Include configure-alertmanager
include_tasks: configure-alertmanager.yml
when: is_alertmanager_enabled | bool
46 changes: 45 additions & 1 deletion ansible/playbooks/roles/prometheus/tasks/upgrade/main.yml
Original file line number Diff line number Diff line change
@@ -125,7 +125,6 @@
- name: Upgrade AlertManager
include_tasks: upgrade/upgrade-alertmanager.yml


# Start services again if needed
- name: Start services
include_tasks: upgrade/start-services.yml
@@ -135,3 +134,48 @@
file:
path: "{{ upgrade_defaults.state_file_path }}"
state: absent

# Can be removed after 1.2 deprecation
- name: Find haproxy-exporter targets
find:
paths: "/etc/prometheus/file_sd"
patterns: "^haproxy-exporter.*.yml$"
use_regex: true
register: haproxy_exporter_filesd

- name: Remove haproxy-exporter targets
file:
path: "{{ item }}"
state: absent
loop: "{{ haproxy_exporter_filesd.files | map(attribute='path') }}"
notify: Restart Prometheus

- name: Check HAproxy
when:
- groups.haproxy is defined
- groups.haproxy | length > 0
block:
- name: Load HAproxy's config vars
include_vars:
file: roles/haproxy/vars/main.yml
name: haproxy_config

- name: Set fact about HAproxy monitoring
set_fact:
haproxy_monitoring_enabled: "{{ haproxy_config.specification.metrics.enable }}"
haproxy_monitoring_port: "{{ haproxy_config.specification.metrics.port }}"

- name: Add HAproxy target to Prometheus config
when: haproxy_monitoring_enabled
blockinfile:
path: /etc/prometheus/prometheus.yml
backup: true
block: |2
- job_name: 'haproxy'
static_configs:
- targets:
{% for haproxy_node in groups['haproxy'] -%}
- "{{ haproxy_node }}:{{ haproxy_monitoring_port }}"
{% endfor %}
insertafter: 'scrape_configs:'
notify: Restart Prometheus
11 changes: 10 additions & 1 deletion ansible/playbooks/roles/prometheus/templates/prometheus.yml.j2
Original file line number Diff line number Diff line change
@@ -50,6 +50,15 @@ scrape_configs:
{% endfor %}
{% endif %}

{% if haproxy_monitoring_enabled is defined and haproxy_monitoring_enabled %}
- job_name: 'haproxy'
static_configs:
- targets:
{% for haproxy_node in groups['haproxy'] %}
- "{{ haproxy_node }}:{{ haproxy_monitoring_port }}"
{% endfor %}
{% endif %}

{% if bearer_token is defined %}
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
@@ -84,7 +93,7 @@ scrape_configs:
insecure_skip_verify: true
bearer_token: "{{ bearer_token }}"
relabel_configs:
- source_labels:
- source_labels:
- __meta_kubernetes_endpoints_name
regex: 'node-exporter'
action: keep
Original file line number Diff line number Diff line change
@@ -95,8 +95,6 @@ python2-cryptography # for Ansible (certificate modules)
python3-3.6.8
quota # for nfs-utils
rabbitmq-server-3.8.9
#rh-haproxy18
#rh-haproxy18-haproxy-syspaths
postgresql13-server
samba-client
samba-client-libs # for samba-client
@@ -156,7 +154,6 @@ https://packages.erlang-solutions.com/erlang/rpm/centos/7/aarch64/esl-erlang_23.
# Grafana package is not downloaded from repository since it was not reliable (issue #2449)
https://dl.grafana.com/oss/release/grafana-8.3.2-1.aarch64.rpm
# --- Exporters ---
https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-arm64.tar.gz
https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-arm64.tar.gz
https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-arm64.tar.gz
@@ -193,8 +190,8 @@ https://grafana.com/api/dashboards/4279/revisions/4/download grafana_dashboard_4
https://grafana.com/api/dashboards/1860/revisions/23/download grafana_dashboard_1860.json
# Kafka Exporter Overview
https://grafana.com/api/dashboards/7589/revisions/5/download grafana_dashboard_7589.json
# HAproxy Servers | HAproxy
https://grafana.com/api/dashboards/367/revisions/3/download grafana_dashboard_367.json
# HaProxy backend (or frontend/servers)
https://grafana.com/api/dashboards/789/revisions/1/download grafana_dashboard_789.json
# Docker and Host Monitoring w/ Prometheus
https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_179.json
# Kubernetes pod and cluster monitoring (via Prometheus)
Original file line number Diff line number Diff line change
@@ -95,8 +95,6 @@ python2-cryptography # for Ansible (certificate modules)
python3-3.6.8
quota # for nfs-utils
rabbitmq-server-3.8.9
rh-haproxy18
rh-haproxy18-haproxy-syspaths
postgresql13-server
repmgr10-5.2.1 # used to upgrade only repmgr
repmgr13-5.2.1
@@ -157,7 +155,6 @@ https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1
# Grafana package is not downloaded from repository since it was not reliable (issue #2449)
https://dl.grafana.com/oss/release/grafana-8.3.2-1.x86_64.rpm
# --- Exporters ---
https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz
https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
@@ -194,8 +191,8 @@ https://grafana.com/api/dashboards/4279/revisions/4/download grafana_dashboard_4
https://grafana.com/api/dashboards/1860/revisions/23/download grafana_dashboard_1860.json
# Kafka Exporter Overview
https://grafana.com/api/dashboards/7589/revisions/5/download grafana_dashboard_7589.json
# HAproxy Servers | HAproxy
https://grafana.com/api/dashboards/367/revisions/3/download grafana_dashboard_367.json
# HaProxy backend (or frontend/servers)
https://grafana.com/api/dashboards/789/revisions/1/download grafana_dashboard_789.json
# Docker and Host Monitoring w/ Prometheus
https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_179.json
# Kubernetes pod and cluster monitoring (via Prometheus)
Original file line number Diff line number Diff line change
@@ -153,7 +153,6 @@ https://github.com/rabbitmq/erlang-rpm/releases/download/v23.1.5/erlang-23.1.5-1
# Grafana package is not downloaded from repository since it was not reliable (issue #2449)
https://dl.grafana.com/oss/release/grafana-8.3.2-1.x86_64.rpm
# --- Exporters ---
https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz
https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
@@ -190,8 +189,8 @@ https://grafana.com/api/dashboards/4279/revisions/4/download grafana_dashboard_4
https://grafana.com/api/dashboards/1860/revisions/23/download grafana_dashboard_1860.json
# Kafka Exporter Overview
https://grafana.com/api/dashboards/7589/revisions/5/download grafana_dashboard_7589.json
# HAproxy Servers | HAproxy
https://grafana.com/api/dashboards/367/revisions/3/download grafana_dashboard_367.json
# HaProxy backend (or frontend/servers)
https://grafana.com/api/dashboards/789/revisions/1/download grafana_dashboard_789.json
# Docker and Host Monitoring w/ Prometheus
https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_179.json
# Kubernetes pod and cluster monitoring (via Prometheus)
Original file line number Diff line number Diff line change
@@ -46,7 +46,6 @@ filebeat 7.9.2
firewalld
fping
gnupg2
haproxy
htop
iftop
jq
@@ -216,7 +215,6 @@ https://packages.elastic.co/curator/5/debian9/pool/main/e/elasticsearch-curator/
# Grafana package is not downloaded from repository since it was not reliable (issue #2449)
https://dl.grafana.com/oss/release/grafana_8.3.2_amd64.deb
# --- Exporters ---
https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
https://github.com/danielqsj/kafka_exporter/releases/download/v1.4.0/kafka_exporter-1.4.0.linux-amd64.tar.gz
https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.16.1/jmx_prometheus_javaagent-0.16.1.jar
https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz
@@ -253,8 +251,8 @@ https://grafana.com/api/dashboards/4279/revisions/4/download grafana_dashboard_4
https://grafana.com/api/dashboards/1860/revisions/23/download grafana_dashboard_1860.json
# Kafka Exporter Overview
https://grafana.com/api/dashboards/7589/revisions/5/download grafana_dashboard_7589.json
# HAproxy Servers | HAproxy
https://grafana.com/api/dashboards/367/revisions/3/download grafana_dashboard_367.json
# HaProxy backend (or frontend/servers)
https://grafana.com/api/dashboards/789/revisions/1/download grafana_dashboard_789.json
# Docker and Host Monitoring w/ Prometheus
https://grafana.com/api/dashboards/179/revisions/7/download grafana_dashboard_179.json
# Kubernetes pod and cluster monitoring (via Prometheus)
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
- name: Remove haproxy_exporter | Stop service
systemd:
state: stopped
name: prometheus-haproxy-exporter

- name: Remove haproxy_exporter | Remove service
file:
path: /etc/systemd/system/prometheus-haproxy-exporter.service
state: absent

- name: Remove haproxy_exporter | Remove directories
file:
path: /opt/haproxy_exporter
state: absent

- name: Remove haproxy_exporter | Remove system user
user:
name: haproxy_exporter
state: absent

- name: Remove haproxy_exporter | Remove system group
group:
name: haproxy_exporter
state: absent
11 changes: 10 additions & 1 deletion ansible/playbooks/roles/upgrade/tasks/load_balancer.yml
Original file line number Diff line number Diff line change
@@ -3,9 +3,18 @@
include_tasks:
file: "legacy-haproxy/{{ ansible_os_family }}/uninstall-haproxy.yml"

- name: load_balancer | Populate service facts
service_facts:
no_log: true

- name: load_balancer | Remove haproxy-exporter
include_tasks:
file: "legacy-haproxy/remove-haproxy-exporter.yml"
when: ansible_facts.services["prometheus-haproxy-exporter.service"] is defined

- name: load_balancer | Upgrade haproxy service (runc)
include_role:
name: haproxy
vars_from: manifest
vars_from: main
vars:
haproxy_upgrade_enabled: true
4 changes: 2 additions & 2 deletions cli/engine/ansible/AnsibleVarsGenerator.py
Original file line number Diff line number Diff line change
@@ -68,15 +68,15 @@ def generate(self):
# is changed between versions (e.g. wal_keep_segments -> wal_keep_size) and sometimes previous parameters
# are not compatible with the new ones, defaults are used for template processing
roles_with_defaults = [
'repository', 'image_registry', 'node_exporter',
'repository', 'image_registry', 'node_exporter', 'haproxy',
'postgresql', 'kafka_exporter', 'jmx_exporter'
]
# now lets add any external configs we want to load
roles_with_defaults = [*roles_with_defaults, *self.inventory_upgrade.get_new_config_roles()]
# In special cases (like haproxy), where user specifies majority of the config, it's easier (and less
# awkward) to re-render config templates instead of modifying (for example with regular expressions)
# no-longer-compatible config files.
roles_with_manifest = ['filebeat', 'haproxy', 'ignite', 'postgresql', 'repository']
roles_with_manifest = ['filebeat', 'ignite', 'postgresql', 'repository']
else:
roles_with_defaults = self.inventory_creator.get_enabled_roles()
roles_with_manifest = [] # applies only to upgrades
3 changes: 1 addition & 2 deletions docs/architecture/logical-view.md
Original file line number Diff line number Diff line change
@@ -17,7 +17,6 @@ Exporter | Component | Description
`Node exporter` | OS/Hardware metrics | [description](https://prometheus.io/docs/guides/node-exporter/)
`Kafka exporter` | Kafka metrics | [description](https://github.com/danielqsj/kafka_exporter)
`JMX exporter` | JVM metrics (Kafka, Zookeeper) | [description](https://github.com/prometheus/jmx_exporter)
`HAProxy exporter` | HAProxy metrics | [description](https://github.com/prometheus/haproxy_exporter)
`cAdvisor` | Container metrics (Kubernetes, Docker) | [description](https://prometheus.io/docs/guides/cadvisor/)

### Prometheus
@@ -114,4 +113,4 @@ Zookeepers are usually deployed in more than one instance - this is called Zooke
`HAProxy` is a high performance load balancer. Applications deployed on `Kubernetes` can be exposed through `HAProxy` that supports TLS termination and supports multiple backends.
Epiphany Platform automates the configuration for backend and frontend of `HAProxy`.

[Read more](http://www.haproxy.org/#desc)
[Read more](http://www.haproxy.org/#desc)
1 change: 1 addition & 0 deletions docs/changelogs/CHANGELOG-1.3.md
Original file line number Diff line number Diff line change
@@ -64,6 +64,7 @@
### Removed

- [#2680](https://github.com/epiphany-platform/epiphany/issues/2680) - Remove PgBouncer standalone installation
- [#1739](https://github.com/epiphany-platform/epiphany/issues/1739) - Replace standalone HAproxy-exporter by embedded one

### Deprecated

5 changes: 2 additions & 3 deletions docs/home/COMPONENTS.md
Original file line number Diff line number Diff line change
@@ -30,7 +30,6 @@ Note that versions are default versions and can be changed in certain cases thro
| Node Exporter | 1.3.1 | https://github.com/prometheus/node_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| Bitnami Node Exporter Helm Chart | 1.1.2 | https://github.com/bitnami/charts | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| Kafka Exporter | 1.4.0 | https://github.com/danielqsj/kafka_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| HAProxy Exporter | 0.10.0 | https://github.com/prometheus/haproxy_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| JMX Exporter | 0.16.1 | https://github.com/prometheus/jmx_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| Postgres Exporter | 0.9.0 | https://github.com/prometheus-community/postgres_exporter | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
| PostgreSQL | 13 | https://www.postgresql.org/ | [PostgreSQL license](http://www.postgresql.org/about/licence/) |
@@ -237,7 +236,7 @@ Note that versions are default versions and can be changed in certain cases thro
| RabbitMQ Monitoring | 4279 | https://grafana.com/grafana/dashboards/4279 | [MIT License](https://github.com/kbudde/rabbitmq_exporter/blob/main/LICENSE) |
| Node Exporter Full | 1860 | https://grafana.com/grafana/dashboards/1860 | [LGPL-3.0 License](https://github.com/rfrail3/grafana-dashboards/blob/master/LICENSE) |
| Kafka Exporter Overview | 7589 | https://grafana.com/grafana/dashboards/7589 | [Apache License 2.0](https://github.com/danielqsj/kafka_exporter/blob/master/LICENSE) |
| HAproxy Servers \| HAproxy | 367 | https://grafana.com/grafana/dashboards/367 | None |
| HaProxy backend (or frontend/servers) | 789 | https://grafana.com/grafana/dashboards/367 | None, [Source Code](https://github.com/tcheronneau/grafana_dashboard) | |
| Docker and Host Monitoring w/ Prometheus | 179 | https://grafana.com/grafana/dashboards/179 | [MIT License](https://github.com/vegasbrianc/prometheus/blob/master/LICENSE) |
| Kubernetes pod and cluster monitoring (via Prometheus) | 6663 | https://grafana.com/grafana/dashboards/6663 | None |
| RabbitMQ cluster monitoring (via Prometheus) | 10991 | https://grafana.com/grafana/dashboards/10991 | None |
| RabbitMQ cluster monitoring (via Prometheus) | 10991 | https://grafana.com/grafana/dashboards/10991 | [License](https://github.com/rabbitmq/rabbitmq-server/blob/master/LICENSE), [Source Code](https://github.com/rabbitmq/rabbitmq-server/blob/master/deps/rabbitmq_prometheus/docker/grafana/dashboards/RabbitMQ-Overview.json) |
5 changes: 2 additions & 3 deletions docs/home/SECURITY.md
Original file line number Diff line number Diff line change
@@ -21,7 +21,6 @@ This setting can be changed in Epiphany yaml configuration files.

Additional to users created by each component Epiphany creates also users and groups:

- haproxy_exporter/haproxy_exporter
- kafka_exporter/kafka_exporter
- node_exporter/node_exporter
- jmx-exporter/jmx-exporter
@@ -123,8 +122,8 @@ different values. The list does not include ports that are bound to the loopback

- 443 - HTTPS frontend
- 9000 - stats page
- unconfigurable random UDP port from ephemeral range* - local connection to rsyslog UDP server (remote access not
needed), see note [[3]](#notes)
- 9101 - metrics
- unconfigurable random UDP port from ephemeral range* - local connection to rsyslog UDP server (remote access not needed), see note [[3]](#notes)

**NOTE:** Not applicable for Ubuntu where UNIX socket is used (deb package's default).

7 changes: 6 additions & 1 deletion docs/home/howto/CLUSTER.md
Original file line number Diff line number Diff line change
@@ -705,10 +705,15 @@ specification:
stats:
enable: true
bind_address: 127.0.0.1:9000
bind_address: 127.0.0.1
port: 9000
uri: "/haproxy?stats"
user: operations
password: your-haproxy-stats-pwd
metrics:
enable: true
bind_address: "*"
port: 9101
frontend:
- name: https_front
port: 443
22 changes: 21 additions & 1 deletion docs/home/howto/MONITORING.md
Original file line number Diff line number Diff line change
@@ -222,7 +222,6 @@ There are many monitoring components deployed with Epiphany that you can visuali
List of monitoring components - so called exporters:

- cAdvisor
- HAProxy Exporter
- JMX Exporter
- Kafka Exporter
- Node Exporter
@@ -330,6 +329,27 @@ specification:

During upgrade Epiphany takes `kibanaserver` (for Kibana) and `logstash` (for Filebeat) user passwords and re-applies them to upgraded configuration of Filebeat and Kibana. Epiphany upgrade of Open Distro, Kibana or Filebeat will fail if `kibanaserver` or `logstash` usernames were changed in configuration of Kibana, Filebeat or Open Distro for Elasticsearch.

# HAProxy

## How to enable HAProxy monitoring

HAProxy metrics are enabled by default. To disable change `specification/metrics/enable` to `false`:

```yaml
kind: configuration/haproxy
title: "HAProxy"
provider: any
name: default
specification:
metrics:
enable: true
bind_address: "*"
port: 9101
```

You can also change the rest of parameters but note, that you would have to change your security group as well.


# RabbitMQ

## How to enable RabbitMQ monitoring
6 changes: 3 additions & 3 deletions schema/aws/defaults/infrastructure/virtual-machine.yml
Original file line number Diff line number Diff line change
@@ -106,8 +106,8 @@ specification:
security:
rules:
- _merge: true
- name: haproxy_exporter
description: Allow haproxy_exporter traffic
- name: haproxy_metrics
description: Allow haproxy_metrics traffic
direction: Inbound
protocol: Tcp
destination_port_range: "9101"
@@ -415,7 +415,7 @@ specification:
protocol: ALL
destination_port_range: "0"
source_address_prefix: "10.1.1.0/24"
destination_address_prefix: "0.0.0.0/0"
destination_address_prefix: "0.0.0.0/0"
---
kind: infrastructure/virtual-machine
title: "Virtual Machine Infra"
5 changes: 2 additions & 3 deletions schema/azure/defaults/infrastructure/virtual-machine.yml
Original file line number Diff line number Diff line change
@@ -425,8 +425,8 @@ specification:
security:
rules:
- _merge: true
- name: haproxy_exporter
description: Allow haproxy_exporter traffic
- name: haproxy_metrics
description: Allow haproxy_metrics traffic
priority: 201
direction: Inbound
access: Allow
@@ -435,7 +435,6 @@ specification:
destination_port_range: "9101"
source_address_prefix: "10.1.0.0/20"
destination_address_prefix: "0.0.0.0/0"

---
kind: infrastructure/virtual-machine
title: "Virtual Machine Infra"
3 changes: 0 additions & 3 deletions schema/common/defaults/configuration/feature-mapping.yml
Original file line number Diff line number Diff line change
@@ -49,8 +49,6 @@ specification:
enabled: true
- name: haproxy
enabled: true
- name: haproxy-exporter
enabled: true
- name: vault
enabled: true
- name: applications
@@ -80,7 +78,6 @@ specification:
- firewall
load_balancer:
- haproxy
- haproxy-exporter
- node-exporter
- filebeat
- firewall
6 changes: 1 addition & 5 deletions schema/common/defaults/configuration/firewall.yml
Original file line number Diff line number Diff line change
@@ -27,11 +27,7 @@ specification:
enabled: true
ports:
- 443/tcp
- 9000/tcp # stats
haproxy_exporter:
enabled: true
ports:
- 9101/tcp
- 9000/tcp # stats and metrics
ignite:
enabled: true
ports:
3 changes: 3 additions & 0 deletions schema/common/defaults/configuration/grafana.yml
Original file line number Diff line number Diff line change
@@ -82,6 +82,9 @@ specification:
# # 1 Node Exporter for Prometheus Dashboard EN v20201010
# - dashboard_id: '11074'
# datasource: 'Prometheus'
# # HaProxy backend (or frontend/servers)
# - dashboard_id: '789'
# datasource: 'Prometheus'

# Get dashboards from https://grafana.com/dashboards. Only for online mode
grafana_online_dashboards: []
14 changes: 0 additions & 14 deletions schema/common/defaults/configuration/haproxy-exporter.yml

This file was deleted.

9 changes: 7 additions & 2 deletions schema/common/defaults/configuration/haproxy.yml
Original file line number Diff line number Diff line change
@@ -10,10 +10,15 @@ specification:

stats:
enable: true
bind_address: 127.0.0.1:9000
bind_address: 127.0.0.1
port: 9000
uri: "/haproxy?stats"
user: operations
password: your-haproxy-stats-pwd
metrics:
enable: true
bind_address: "*"
port: 9101
frontend:
- name: https_front
port: 443
@@ -27,4 +32,4 @@ specification:
# servers: # Definition for server to that hosts the application.
# - name: "node1"
# address: "epiphany-vm1.domain.com"
port: 30104
port: 30104
9 changes: 0 additions & 9 deletions schema/common/validation/configuration/firewall.yml
Original file line number Diff line number Diff line change
@@ -53,15 +53,6 @@ properties:
type: array
items:
type: string
haproxy_exporter:
type: object
properties:
enabled:
type: boolean
ports:
type: array
items:
type: string
ignite:
type: object
properties:
25 changes: 0 additions & 25 deletions schema/common/validation/configuration/haproxy-exporter.yml

This file was deleted.

11 changes: 11 additions & 0 deletions schema/common/validation/configuration/haproxy.yml
Original file line number Diff line number Diff line change
@@ -20,12 +20,23 @@ properties:
type: boolean
bind_address:
type: string
port:
type: integer
uri:
type: string
user:
type: string
password:
type: string
metrics:
type: object
properties:
enable:
type: boolean
bind_address:
type: string
port:
type: integer
frontend:
type: array
items:
23 changes: 21 additions & 2 deletions tests/spec/spec/haproxy/haproxy_spec.rb
Original file line number Diff line number Diff line change
@@ -3,8 +3,9 @@
haproxy_host = 'localhost'
haproxy_front_port = 443
haproxy_stats_port = 9000
haproxy_metrics_port = 9101

# Running systemctl status command as "is-active" returns "unknown" in result
# Running systemctl status command as "is-active" returns "unknown" in result
# https://bugzilla.redhat.com/show_bug.cgi?id=1073481

describe 'Checking HAProxy service status' do
@@ -37,7 +38,10 @@
describe port(haproxy_stats_port) do
it { should be_listening }
end
end
describe port(haproxy_metrics_port) do
it { should be_listening }
end
end

describe 'Checking SSL certificates' do
describe file('/etc/ssl/haproxy') do
@@ -84,3 +88,18 @@
end
end

describe 'Checking HAProxy HTTP status code for metrics page' do
describe command("curl -k -o /dev/null -s -w '%{http_code}' http://#{haproxy_host}:#{haproxy_metrics_port}/metrics") do
it "is expected to be equal" do
expect(subject.stdout.to_i).to eq 200
end
end
end

describe 'Checking if it is possible to collect the metrics from HAProxy' do
describe command("curl -s http://#{haproxy_host}:#{haproxy_metrics_port}/metrics") do
its(:stdout) { should include "haproxy_process_nbthread" }
its(:stdout) { should include "haproxy_process_current_tasks" }
its(:exit_status) { should eq 0 }
end
end
42 changes: 0 additions & 42 deletions tests/spec/spec/haproxy_exporter/haproxy_exporter_spec.rb

This file was deleted.

35 changes: 18 additions & 17 deletions tests/spec/spec/prometheus/prometheus_spec.rb
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
alertmanager_host = 'localhost'
alertmanager_port = 9093
kubelet_port = 10250
haproxy_metrics_port = 9101

describe 'Check if Prometheus user exists' do
describe group('prometheus') do
@@ -101,24 +102,24 @@
end
end

describe 'Check configuration files for HAProxy Exporter' do
listInventoryHosts("haproxy_exporter").each do |val|
describe command("ls /etc/prometheus/file_sd") do
describe 'Check Prometheus configuration if exist HAproxy target' do
listInventoryHosts("haproxy").each do |val|
describe command("cat /etc/prometheus/prometheus.yml") do
let(:disable_sudo) { false }
its(:stdout) { should match /haproxy-exporter-#{val}.yml/ }
its(:stdout) { should include "#{val}:#{haproxy_metrics_port}" }
end
end
end

describe 'Check connection to HAProxy Exporter hosts' do
listInventoryHosts("haproxy_exporter").each do |val|
let(:disable_sudo) { false }
describe command("curl -o /dev/null -s -w '%{http_code}' $(grep -oP \"(?<=targets: \\\[\').*(?=\'\\\])\" /etc/prometheus/file_sd/haproxy-exporter-#{val}.yml)/metrics") do
it "is expected to be equal" do
expect(subject.stdout.to_i).to eq 200
describe 'Check connection HAproxy metrics endpoint' do
listInventoryHosts("haproxy").each do |val|
let(:disable_sudo) { false }
describe command("curl -o /dev/null -s -w '%{http_code}' #{val}:#{haproxy_metrics_port}/metrics") do
it "is expected to be equal" do
expect(subject.stdout.to_i).to eq 200
end
end
end
end
end
end

describe 'Check configuration files for JMX Exporter' do
@@ -257,7 +258,7 @@
# its(:stdout) { should_not match /FAILED/ }
# its(:exit_status) { should eq 0 }
# end
# end
# end

# describe 'Check if it is possible to create a rule checking if node is up' do
# describe command("cp -p /etc/prometheus/rules/UpDown.rules /etc/prometheus/rules/TEST_RULE.rules && sed -i 's/UpDown/TEST_RULE/g; s/down/up/g; s/== 0/== 1/g; \
@@ -279,8 +280,8 @@
# its(:stdout) { should match /READY/ }
# its(:exit_status) { should eq 0 }
# end
# end
# end

# # Tests for Alertmanager assuming monitoring.alerts.enable == true and monitoring.alerts.handlers.mail.enable == true

# if readDataYaml["monitoring"]["alerts"]["handlers"]["mail"]["enable"] == true
@@ -290,7 +291,7 @@
# let(:disable_sudo) { false }
# it { should be_listening }
# end
# end
# end

# describe 'Check if Alertmanager service is running' do
# describe service('alertmanager') do
@@ -319,7 +320,7 @@
# describe command("curl -XPOST -d '[{\"labels\":{\"alertname\":\"TEST ALERT\", \"severity\":\"critical\"}}]' #{alertmanager_host}:#{alertmanager_port}/api/v1/alerts") do
# its(:stdout_as_json) { should include('status' => 'success') }
# end
# end
# end

# end
# end

0 comments on commit 12b99bf

Please sign in to comment.