Upgrade Grafana to v7.3.5 [Backport] (#2020)

Upgrade Grafana to v7.3.5 [Backport to v0.8]

CHANGELOG-0.8.md

@@ -15,6 +15,7 @@
 - [#1964](https://github.com/epiphany-platform/epiphany/issues/1964) - Upgrade Elasticsearch Curator to v5.8.3
 - [#1919](https://github.com/epiphany-platform/epiphany/issues/1919) - Upgrade Kafka to v2.6.0
 - [#1926](https://github.com/epiphany-platform/epiphany/issues/1926) - Upgrade Zookeeper to v3.5.8
+- [#1949](https://github.com/epiphany-platform/epiphany/issues/1949) - Upgrade Grafana to v7.3.5
 - [#1855](https://github.com/epiphany-platform/epiphany/issues/1855) - Upgrade Docker to v19.03.14
 
 ## [0.8.0] 2020-10-22

core/src/epicli/data/common/ansible/playbooks/grafana.yml

@@ -1,7 +1,7 @@
 ---
 - hosts: all
   gather_facts: yes
-  tasks: [ ]  
+  tasks: []  
 
 - hosts: grafana
   become: true

core/src/epicli/data/common/ansible/playbooks/roles/grafana/defaults/main.yml

@@ -1,8 +1,5 @@
 ---
-grafana_version: "{{ specification.version }}"
-
-# Should we use the provisioning capability when possible (provisioning require grafana >= 5.0)
-grafana_use_provisioning: "{{ specification.grafana_use_provisioning }}"
+grafana_version: 7.3.5
 
 # Should the provisioning be kept synced. If true, previous provisioned objects will be removed if not referenced anymore.
 grafana_provisioning_synced: "{{ specification.grafana_provisioning_synced }}"
@@ -93,4 +90,3 @@ grafana_api_keys: "{{ specification.grafana_api_keys }}"
 grafana_api_keys_dir: "{{ lookup('env', 'HOME') }}/grafana/keys"
 
 grafana_environment: {}
-

core/src/epicli/data/common/ansible/playbooks/roles/grafana/handlers/main.yml

@@ -1,27 +1,23 @@
 ---
 - name: restart grafana
-  become: true
   service:
     name: grafana-server
     state: restarted
-  tags:
-    - grafana_run
 
 - name: Set privileges on provisioned dashboards
-  become: true
   file:
     path: "{{ grafana_data_dir }}/dashboards"
     recurse: true
     owner: grafana
     group: grafana
-    mode: 0640
-  listen: "provisioned dashboards changed"
+    mode: u=rw,g=r,o=
+  listen: provisioned dashboards changed
 
 - name: Set privileges on provisioned dashboards directory
   become: true
   file:
     path: "{{ grafana_data_dir }}/dashboards"
     state: directory
     recurse: false
-    mode: 0755
-  listen: "provisioned dashboards changed"
+    mode: u=rwx,go=rx
+  listen: provisioned dashboards changed

core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/api_keys.yml

@@ -1,10 +1,11 @@
 ---
 - name: Ensure grafana key directory exists
+  delegate_to: localhost
   become: false
   file:
     path: "{{ grafana_api_keys_dir }}/{{ inventory_hostname }}"
     state: directory
-  delegate_to: localhost
+    mode: u=rwx,go=
 
 - name: Check api key list
   uri:
@@ -13,8 +14,8 @@
     password: "{{ grafana_security.admin_password }}"
     force_basic_auth: true
     return_content: true
-  no_log: true
   register: existing_api_keys
+  no_log: true
 
 - name: Create grafana api keys
   uri:
@@ -25,17 +26,18 @@
     method: POST
     body_format: json
     body: "{{ item | to_json }}"
-  with_items: "{{ grafana_api_keys }}"
-  no_log: true
-  when: ((existing_api_keys['json'] | selectattr("name", "equalto", item['name'])) | list) | length == 0
+  when: existing_api_keys.json | selectattr('name', '==', item.name) | length == 0
+  loop: "{{ grafana_api_keys }}"
   register: new_api_keys
+  no_log: true
 
 - name: Create api keys file to allow the keys to be seen and used by other automation
+  delegate_to: localhost
   become: false
   copy:
-    dest: "{{ grafana_api_keys_dir }}/{{ inventory_hostname }}/{{ item['item']['name'] }}.key"
-    content: "{{ item['json']['key'] }}"
+    dest: "{{ grafana_api_keys_dir }}/{{ inventory_hostname }}/{{ item.item.name }}.key"
+    content: "{{ item.json.key }}"
     backup: false
-  when: item['json'] is defined
-  with_items: "{{ new_api_keys['results'] }}"
-  delegate_to: localhost
+    mode: u=rw,go=
+  when: item.json is defined
+  loop: "{{ new_api_keys.results }}"

core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/configure.yml

@@ -5,45 +5,67 @@
     state: directory
     owner: root
     group: grafana
-  with_items:
-    - "/etc/grafana"
-    - "/etc/grafana/datasources"
-    - "/etc/grafana/provisioning"
-    - "/etc/grafana/provisioning/datasources"
+    mode: ug=rwx,o=
+  loop:
+    - /etc/grafana/datasources
+    - /etc/grafana/provisioning/datasources
+    - /etc/grafana/ssl
+
+- name: Generate self signed SSL certificates
+  command: >
+    openssl req
+      -new
+      -newkey rsa:4096
+      -days 365
+      -nodes
+      -x509
+      -subj "/C=US/ST=NY/L=NY/O=NA/CN=localhost"
+      -keyout /etc/grafana/ssl/grafana_key.key
+      -out /etc/grafana/ssl/grafana_cert.pem
+  args:
+    creates: /etc/grafana/ssl/grafana_cert.pem
+
+- name: Ensure grafana ssl directory permissions are correct
+  file:
+    path: /etc/grafana/ssl
+    state: directory
+    owner: root
+    group: grafana
+    mode: ug+rw,o=
+    recurse: true
 
 - name: Create grafana main configuration file
   template:
     src: grafana.ini.j2
     dest: /etc/grafana/grafana.ini
     owner: root
     group: grafana
-    mode: 0640
-  no_log: true
+    mode: ug=rw,o=
   notify: restart grafana
+  no_log: true
 
 - name: Create grafana LDAP configuration file
   template:
     src: ldap.toml.j2
     dest: "{{ grafana_auth.ldap.config_file | default('/etc/grafana/ldap.toml') }}"
     owner: root
     group: grafana
-    mode: 0640
+    mode: ug=rw,o=
   when:
     - "'ldap' in grafana_auth"
     - "'enabled' not in grafana_auth.ldap or grafana_auth.ldap.enabled"
-  no_log: true
   notify: restart grafana
+  no_log: true
 
 - name: Create grafana directories
   file:
     path: "{{ item }}"
     state: directory
-    mode: 0755
-    owner: "grafana"
-    group: "grafana"
+    owner: grafana
+    group: grafana
+    mode: ug=rwx,o=
   with_items:
     - "{{ grafana_logs_dir }}"
-    - "{{ grafana_data_dir }}"
     - "{{ grafana_data_dir }}/dashboards"
     - "{{ grafana_data_dir }}/plugins"
 
@@ -59,6 +81,6 @@
 - name: Enable and start Grafana systemd unit
   systemd:
     name: grafana-server
-    enabled: true
     state: started
-    daemon_reload: true
+    enabled: true
+    daemon_reload: true

core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/dashboards.yml

@@ -1,34 +1,25 @@
 ---
-- become: false
+- name: Download and prepare dashboards
   delegate_to: localhost
+  become: false
   run_once: true
   block:
     - name: Create local grafana dashboard directory
       tempfile:
         state: directory
-      register: _tmp_dashboards
+      register: tmp_dashboards
       changed_when: false
       check_mode: false
 
-    # Use curl to solve issue #77
-    - name: download grafana dashboard from grafana.net to local directory
-      command: >
-        curl --fail --compressed
-        https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download
-        -o {{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json
-      args:
-        creates: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json"
-        warn: false
-      register: _download_dashboards
-      until: _download_dashboards is succeeded
+    - name: Download grafana dashboard from grafana.com to local directory
+      get_url:
+        url: https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download
+        dest: "{{ tmp_dashboards.path }}/{{ item.dashboard_id }}.json"
+      register: result
+      until: result is success
       retries: 5
       delay: 2
-      with_items: "{{ grafana_dashboards }}"
-      when: grafana_dashboards | length > 0
-      changed_when: false
-      check_mode: false
-      tags:
-        - skip_ansible_lint
+      loop: "{{ grafana_dashboards }}"
 
     # As noted in [1] an exported dashboard replaces the exporter's datasource
     # name with a representative name, something like 'DS_GRAPHITE'. The name
@@ -61,56 +52,19 @@
     #
     # This regex can be tested and understood better by looking at the
     # matches and non-matches in https://regex101.com/r/f4Gkvg/6
-
     - name: Set the correct data source name in the dashboard
       replace:
-        dest: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json"
+        dest: "{{ tmp_dashboards.path }}/{{ item.dashboard_id }}.json"
         regexp: '"(?:\${)?DS_[A-Z0-9_-]+(?:})?"'
         replace: '"{{ item.datasource }}"'
       changed_when: false
-      with_items: "{{ grafana_dashboards }}"
-      when: grafana_dashboards | length > 0
+      loop: "{{ grafana_dashboards }}"
 
-- name: Import grafana dashboards through API
-  uri:
-    url: "{{ grafana_api_url }}/api/dashboards/db"
-    user: "{{ grafana_security.admin_user }}"
-    password: "{{ grafana_security.admin_password }}"
-    force_basic_auth: true
-    method: POST
-    body_format: json
-    body: >
-      {
-        "dashboard": {{ lookup("file", item) }},
-        "overwrite": true,
-        "message": "Updated by ansible"
-      }
-  no_log: true
-  with_fileglob:
-    - "{{ _tmp_dashboards.path }}/*"
-    - "{{ grafana_dashboards_dir }}/*.json"
-  when: not grafana_use_provisioning
-
-# TODO: uncomment this when ansible 2.7 will be min supported version
-# - name: import grafana dashboards
-#   grafana_dashboard:
-#     grafana_url: "{{ grafana_api_url }}"
-#     grafana_user: "{{ grafana_security.admin_user }}"
-#     grafana_password: "{{ grafana_security.admin_password }}"
-#     path: "/tmp/dashboards/{{ item }}"
-#     message: Updated by ansible
-#     state: present
-#     overwrite: true
-#   no_log: true
-#   with_fileglob:
-#     - "/tmp/dashboards/*"
-
-- when: grafana_use_provisioning
+- name: Update dashboards
   block:
     - name: Create/Update dashboards file (provisioning)
-      become: true
       copy:
-        dest: "/etc/grafana/provisioning/dashboards/ansible.yml"
+        dest: /etc/grafana/provisioning/dashboards/ansible.yml
         content: |
           apiVersion: 1
           providers:
@@ -123,7 +77,7 @@
         backup: false
         owner: root
         group: grafana
-        mode: 0640
+        mode: u=rw,g=r,o=
       notify: restart grafana
 
     - name: Register previously copied dashboards
@@ -132,30 +86,28 @@
         hidden: true
         patterns:
           - "*.json"
-      register: _dashboards_present
+      register: dashboards_present
       when: grafana_provisioning_synced
 
     - name: Import grafana dashboards
-      become: true
       copy:
         src: "{{ item }}"
         dest: "{{ grafana_data_dir }}/dashboards/{{ item | basename }}"
       with_fileglob:
-        - "{{ _tmp_dashboards.path }}/*"
+        - "{{ tmp_dashboards.path }}/*.json"
         - "{{ grafana_dashboards_dir }}/*.json"
-      register: _dashboards_copied
-      notify: "provisioned dashboards changed"
+      register: dashboards_copied
+      notify: provisioned dashboards changed
 
     - name: Get dashboard lists
       set_fact:
-        _dashboards_present_list: "{{ _dashboards_present | json_query('files[*].path') | default([]) }}"
-        _dashboards_copied_list: "{{ _dashboards_copied | json_query('results[*].dest') | default([]) }}"
+        dashboards_present_list: "{{ dashboards_present.files | map(attribute='path') | list }}"
+        dashboards_copied_list: "{{ dashboards_copied.results | map(attribute='dest') | list }}"
       when: grafana_provisioning_synced
 
     - name: Remove dashbards not present on deployer machine (synchronize)
-      become: true
       file:
         path: "{{ item }}"
         state: absent
-      with_items: "{{ _dashboards_present_list | difference( _dashboards_copied_list ) }}"
-      when: grafana_provisioning_synced
+      loop: "{{ dashboards_present_list | difference(dashboards_copied_list) }}"
+      when: grafana_provisioning_synced

core/src/epicli/data/common/ansible/playbooks/roles/grafana/tasks/datasources.yml

@@ -1,23 +1,7 @@
 ---
-- name: Ensure datasources exist (via API)
-  grafana_datasource:
-    grafana_url: "{{ grafana_api_url }}"
-    grafana_user: "{{ grafana_security.admin_user }}"
-    grafana_password: "{{ grafana_security.admin_password }}"
-    name: "{{ item.name }}"
-    url: "{{ item.url }}"
-    ds_type: "{{ item.type }}"
-    access: "{{ item.access | default(omit) }}"
-    is_default: "{{ item.isDefault | default(omit) }}"
-    basic_auth_user: "{{ item.basicAuthUser | default(omit) }}"
-    basic_auth_password: "{{ item.basicAuthPassword | default(omit) }}"
-  with_items: "{{ grafana_datasources }}"
-  when: not grafana_use_provisioning
-
 - name: Create/Update datasources file (provisioning)
-  become: true
   copy:
-    dest: "/etc/grafana/provisioning/datasources/ansible.yml"
+    dest: /etc/grafana/provisioning/datasources/ansible.yml
     content: |
       apiVersion: 1
       deleteDatasources: []
@@ -26,6 +10,5 @@
     backup: false
     owner: root
     group: grafana
-    mode: 0640
+    mode: u=rw,g=r,o=
   notify: restart grafana
-  when: grafana_use_provisioning