Update Grafana to version 11 and provision dashboards from usegalaxy-eu/grafana-dashboards

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "mounts"]
 	path = mounts
 	url = https://github.com/usegalaxy-eu/mounts
+[submodule "files/grafana"]
+	path = files/grafana
+	url = https://github.com/usegalaxy-eu/grafana-dashboards.git

grafana.yml

@@ -1,54 +1,96 @@
 ---
 - name: Grafana
   hosts: grafana
-  become: true
-  vars:
-    hostname: stats.galaxyproject.eu
   vars_files:
-    - group_vars/grafana-gitter-bridge.yml
     - secret_group_vars/all.yml
-  # This shouldn't be necessary? but if certs fail, then do this. This is
-  # *supposed* to be properly handled by the nginx role, but the permission
-  # change is likely only applied if a change is detected since it can't figure
-  # out if the rule is there or not.
-  # TODO: make the nginx task check if the rule is in place, rather than a change in path.
-  # pre_tasks:
-  #   - name: Put SELinux in permissive mode, logging actions that would be blocked.
-  #     selinux:
-  #       policy: targeted
-  #       state: permissive
   pre_tasks:
+    #- name: Put SELinux in permissive mode, logging actions that would be blocked.
+    #  # Putting SELinux in permissive mode should not be necessary. But if
+    #  # certs fail, then do it. It is supposed to be properly handled by
+    #  # the `galaxyproject.nginx` role, but the permission change is likely
+    #  # only applied if a change is detected since it can't figure out if
+    #  # the rule is there or not.
+    #  # TODO: make the nginx task check if the rule is in place, rather than a change in path.
+    #  become: true
+    #  ansible.posix.selinux:
+    #    policy: targeted
+    #    state: permissive
     - name: Set default version of Python
-      alternatives:
+      become: true
+      community.general.alternatives:
         name: python
         path: /usr/bin/python3
     - name: Install Dependencies
-      package:
+      become: true
+      ansible.builtin.package:
         name: ['python3-virtualenv']
+    - name: Ensure git is installed. (hxr.monitor-ssl)
       become: true
-    - name: Disable firewalld service
-      ansible.builtin.service:
-        name: firewalld
-        enabled: false
-        state: stopped
+      ansible.builtin.package:
+        name:
+          - git
   collections:
     - devsec.hardening
+    - grafana.grafana
   roles:
     ## Starting configuration of the operating system
     - role: usegalaxy_eu.handy.os_setup
+      become: true
       vars:
+        hostname: "{{ grafana_domain }}"
         enable_hostname: true
-        enable_powertools: true        # geerlingguy.repo-epel role doesn't enable PowerTools repository
-    - geerlingguy.repo-epel     # Install EPEL repository
-    - usegalaxy-eu.autoupdates  # keep all of our packages up to date
-    - influxdata.chrony         # Keep our time in sync.
+        enable_powertools: true         # geerlingguy.repo-epel role doesn't enable PowerTools repository
+    - role: geerlingguy.repo-epel     # Install EPEL repository
+      become: true
+    - role: usegalaxy-eu.autoupdates  # keep all of our packages up to date
+      become: true
+      vars:
+        hostname: "{{ grafana_domain }}"
+    - influxdata.chrony               # Keep our time in sync.
+
     ## Monitoring
-    - hxr.monitor-ssl
-    - hxr.monitor-email
+    - dj-wasabi.telegraf
+    - role: hxr.monitor-ssl
+      become: true
+    - role: hxr.monitor-email
+      become: true
 
-    - galaxyproject.nginx
-    - cloudalchemy.grafana
-    - pgs
-    - hxr.grafana-gitter-bridge
+    ## Grafana
+    - role: galaxyproject.nginx
+      become: true
+    - grafana
+    - role: pgs
+      become: true
+    - role: hxr.grafana-gitter-bridge
+      become: true
     - usegalaxy_eu.grafana_matrix_forwarder
-    - dj-wasabi.telegraf
+
+  post_tasks:
+    # The `[unified_alerting]` section of grafana.ini is not populated by the
+    # `grafana.grafana.grafana` role yet. It will be when PR [1] is merged. In
+    # the meantime, it is populated with this post-task.
+    #
+    # References:
+    # - [1] https://github.com/grafana/grafana-ansible-collection/pull/215
+    - name: Write Grafana unified alerting settings to grafana.ini (grafana.grafana.grafana)
+      become: true
+      community.general.ini_file:
+        path: /etc/grafana/grafana.ini
+        section: unified_alerting
+        option: "{{ item.key }}"
+        value: "{{ item.value }}"
+        state: present
+        owner: "root"  # copied from `grafana.grafana.grafana` v5.2.0
+        group: "grafana"  # copied from `grafana.grafana.grafana` v5.2.0
+        mode: "0640"  # copied from `grafana.grafana.grafana` v5.2.0
+      loop: "{{ grafana_unified_alerting | default({}) | dict2items }}"
+
+    - name: Open nginx ports
+      become: true
+      ansible.posix.firewalld:
+        port: "{{ item }}"
+        permanent: true
+        state: enabled
+      with_items:
+        - 80/tcp
+        - 443/tcp

group_vars/grafana.yml → group_vars/grafana/vars.yml

@@ -35,22 +35,28 @@ nginx_conf_ssl_certificate: /etc/ssl/certs/fullchain.pem
 nginx_conf_ssl_certificate_key: /etc/ssl/user/privkey-nginx.pem
 
 # Grafana
-grafana_data_dir: /data/monitoring/grafana_data
-grafana_instance: "{{ hostname }}"
+grafana_version: 11.0.0
+
 grafana_address: "127.0.0.1"
-grafana_url: "https://stats.galaxyproject.eu"
-grafana_server:
-  domain: "{{ hostname }}"
-  root_url: "https://%(domain)s/"
+grafana_domain: stats.galaxyproject.eu
+grafana_url: "https://{{ grafana_domain }}"
 
 grafana_users:
   default_theme: "light"
 
 grafana_security:
   admin_user: admin
-  admin_password: "{{ grafana_auth_admin_password }}"
+  admin_password: "{{ vault_grafana_auth_admin_password }}"
   allow_embedding: "true"
   cookie_secure: "true"
+grafana_security_secret_key: "{{ vault_grafana_security_secret_key }}"
+
+grafana_database:
+  type: postgres
+  host: sn05.galaxyproject.eu:5432
+  name: "{{ vault_grafana_database_name }}"
+  user: "{{ vault_grafana_database_user }}"
+  password: "{{ vault_grafana_database_password }}"
 
 grafana_auth:
   anonymous:
@@ -61,25 +67,48 @@ grafana_auth:
   github:
     enabled: "true"
     allow_sign_up: "true"
-    client_id: "{{ grafana_auth__github_client_id }}"
-    client_secret: "{{ grafana_auth__github_client_secret }}"
+    client_id: "{{ vault_grafana_auth__github_client_id }}"
+    client_secret: "{{ vault_grafana_auth__github_client_secret }}"
     scopes: "user:email,read:org"
     auth_url: "https://github.com/login/oauth/authorize"
     token_url: "https://github.com/login/oauth/access_token"
     api_url: "https://api.github.com/user"
     allowed_organisations: "usegalaxy.eu galaxyproject"
+grafana_auth_admin_password: "{{ vault_grafana_auth_admin_password }}"
 
 grafana_image_storage:
   provider: local
 
-grafana_environment:
-  GF_PANELS_DISABLE_SANITIZE_HTML: 'true'
+grafana_panels:
+  disable_sanitize_html: "true"
+
+# This setting is not yet used by the role `grafana.grafana.grafana`. It will
+# be when PR [1] is merged.
+#
+# References:
+# - [1] https://github.com/grafana/grafana-ansible-collection/pull/215
+grafana_unified_alerting:
+  enabled: "true"
+  execute_alerts: "true"
+
+# Legacy alerting was removed in Grafana 11 [1]. However, the Ansible role
+# `grafana.grafana.grafana` still populates the [alerting] section [2] unless
+# `grafana_alerting` is empty. The setting can be removed from this file as
+# soon as PR [3] is merged.
+#
+# References:
+# - [1] https://grafana.com/blog/2024/04/04/legacy-alerting-removal-what-you-need-to-know-about-upgrading-to-grafana-alerting/
+# - [2] https://github.com/grafana/grafana-ansible-collection/blob/2e7fd0591d8ad1700186174213b8142047525b88/roles/grafana/templates/grafana.ini.j2#L116-L127
+# - [3] https://github.com/grafana/grafana-ansible-collection/pull/215
+grafana_alerting: {}
 
 grafana_plugins:
   - grafana-worldmap-panel
   - grafana-piechart-panel
   - natel-discrete-panel
 
+grafana_dashboards_dir: "files/grafana"
+
 grafana_datasources:
   - name: Galaxy
     type: influxdb
@@ -90,8 +119,8 @@ grafana_datasources:
     editable: false
     database: galaxy
     user: "{{ influxdb.grafana.username }}"
-    secureJsonData:
-      password: "{{ influxdb.grafana.password }}"
+    password: "{{ influxdb.grafana.password }}"
+    uid: P9B81C0353945995B
   - name: Galaxy Historical
     type: influxdb
     access: proxy
@@ -100,8 +129,8 @@ grafana_datasources:
     editable: false
     database: galaxy_stats
     user: "{{ influxdb.grafana.username }}"
-    secureJsonData:
-      password: "{{ influxdb.grafana.password }}"
+    password: "{{ influxdb.grafana.password }}"
+    uid: PEBD82B4560F292BD
   - name: GRT Tool Usage
     type: influxdb
     access: proxy
@@ -110,8 +139,8 @@ grafana_datasources:
     editable: false
     database: grt
     user: "{{ influxdb.grafana.username }}"
-    secureJsonData:
-      password: "{{ influxdb.grafana.password }}"
+    password: "{{ influxdb.grafana.password }}"
+    uid: PA4245ACF5D5D4D2B
   - name: InfluxDB - internal
     type: influxdb
     access: proxy
@@ -120,8 +149,8 @@ grafana_datasources:
     editable: false
     database: _internal
     user: "{{ influxdb.grafana.username }}"
-    secureJsonData:
-      password: "{{ influxdb.grafana.password }}"
+    password: "{{ influxdb.grafana.password }}"
+    uid: P97EDD1499B7D2981
   - name: Notifications
     type: influxdb
     access: proxy
@@ -130,8 +159,8 @@ grafana_datasources:
     editable: false
     database: notifications
     user: "{{ influxdb.grafana.username }}"
-    secureJsonData:
-      password: "{{ influxdb.grafana.password }}"
+    password: "{{ influxdb.grafana.password }}"
+    uid: P788011833A5A0F22
 
 hxr_monitor_ssl_expiry:
   # A couple of usegalaxy.eu subdomains.
@@ -172,9 +201,9 @@ telegraf_plugins_extra:
 
 #galaxy_test_url: https://usegalaxy.eu
 #galaxy_test_user:
-    #api_key: "{{ galaxy_test_user__api_key }}"
+    #api_key: "{{ vault_galaxy_test_user__api_key }}"
     #username: [email protected]
-    #password: "{{ galaxy_test_user__password }}"
+    #password: "{{ vault_galaxy_test_user__password }}"
     #handlers:
         #- "handler0"
         #- "handler1"
@@ -219,4 +248,20 @@ galaxy_nagios_urls:
       url: "https://usegalaxy.eu/phinch/index.html"
       code: 200
 
-matrix_forwarder_resolve_mode: reaction
+
+# Grafana Gitter Bridge
+ggb_create_user: true
+ggb_create_group: true
+bridge_token: "{{ vault_grafana_gitter_bridge_token }}"
+gitter_auth_token: "{{ vault_grafana_gitter_bridge_auth_token }}"
+gitter_room_id: "{{ vault_grafana_gitter_room_id }}"
+
+# Grafana Matrix Forwarder
+grafana_matrix_forwarder_version: "0.6.0"
+grafana_matrix_forwarder_user: centos
+grafana_matrix_forwarder_group: centos
+grafana_matrix_forwarder_homeserver: https://matrix-client.matrix.org
+grafana_matrix_forwarder_port: 6000
+grafana_matrix_forwarder_username: "{{ vault_grafana_matrix_forwarder_username }}"
+grafana_matrix_forwarder_password: "{{ vault_grafana_matrix_forwarder_password }}"
+grafana_matrix_forwarder_resolve_mode: reaction

group_vars/grafana/vault.yml

@@ -0,0 +1,51 @@
+$ANSIBLE_VAULT;1.1;AES256
+36613934353635356239613433626262353136393732346330383338316335656361306332653465
+3864666638366335346431316261643762333531363434350a333061613863353038633938376638
+32653139363938616234626463643734336366666334643031306530353162653937393865303161
+6463356666333862660a303364623139366235633661333864383034393366363036613064396435
+61613031663165613439343934633338383261626531396137626564323862363935666535633637
+32393065386135663566623231333761386434373238363434393930333334633330333366613638
+31326133383065376461653165303533333836323230643264316634626231613436343932383038
+66653562666333346338363736316430613630393563383538373037613533653037633833633832
+39363738646335646232306431626336613738653630306232313361643536356233346330643837
+33376437636637383637656531316238303132376136613536323736333161316334366234383331
+32373431323463363330633966346137626533613633663431656162306436393263376264343062
+36636136656135356666313531346331373461663236663432333166363836636432653836663231
+66306264663931336131383832396631616331306466653362313232346365626236616262646336
+33396235396435393638373666363261316265663431396462313130386561363461666532343265
+31626631623839306338363438656565393830623362653235363964313962663361626664396436
+30636331343835633430633633366162643364643136626161336663393939333539386361663064
+38303363363138623361353866396563336335663965656333353937633235623563363433316637
+64313566386464346434643135303061373361616431396630613934646161646130326132373831
+31363330303132633936353166303635336337626535613139376537386363373537346238636339
+36643039346637356336633738323966346666316164343339633864303331613230323965633463
+38303436336139303663333866363961663862356465616533663563313666646532363835653935
+38653665356231373161613036316537333166373934343365653536653933666336396461656230
+31633330393232626164343365313765303837393164613232626339656234313430643565613465
+37656432393866336465623461356131393966343130326232373639366264313966623733373739
+30393966663931613430636333343537316337666162333537323861303762386261383739656639
+30636564353130666162353037666633656336306337336238353936303135336530343436653134
+65643665366564383264333734636234393562323934633731363961626132306337313335323662
+37353066636161303761316364636164326636333532623966343339323665643438313833316538
+61346163633661356561353335333739393333666162396661623662633461323535383633663538
+30303364346466393165626264303331313730336433306233356532643533373266333338336534
+36313061636665666334356238386131353938363031653662303764376537333635306262383432
+63353034343038653137303564643465626136383135333831663334333362613632303332663536
+36633961393862313864363262353866353733643735326131303563386563303236353262646633
+37326236333466643834636633663630646264643265653263653330346638653663343566313264
+35626631326231386265333763316532313439636339386633363535626233633830636661613965
+30646337343430376563313665386431613034333734383231363134393132643938623931666365
+61303635653062343664653135323934323564303933343037383632363231336533323038333666
+34393438326635643837366232663362313536633634323962663637366630313438336131623135
+35646662393431336538376637363962393266336664383265343034653662623161336166343366
+33346262666138386630656331613331666539343865303936613138343537633666613861616139
+61663533383066323137323161376464336663373131303566373734363238336565623665316165
+32656465336333363161356164663734656362663135666237633364306339313537363032303435
+36336130616330343838663431376561343562393438313134616133343237643363396532353338
+36343266326462313336396461353033643838613438366637633837653638336237626633356339
+32306431663336383461646236636433623639363732353838356431353832663434343566326564
+64313836393139323038616435343266313261373833663034633733316161343739326531393437
+66323961383262636333616363373565393837623330303630383132373338383066383764666365
+61366165666263396263613462646535643031663863323764613139623630653231636239386634
+31663463353835373561653964346434346662373661646465633339323238653861666332663036
+3964356164653164626236376335613430643562666639323136

requirements.txt

@@ -1,3 +1,4 @@
 ansible-core==2.14.5
 boto==2.49.0
+jmespath>=0.9.0,<2  # (grafana.grafana.grafana)
 docker>=5.0.0