Feature: Add Puppet Classes view

Dockerfile

@@ -8,9 +8,10 @@ EXPOSE 80
 
 HEALTHCHECK --interval=1m --timeout=5s --start-period=10s CMD python3 -c "import requests; import sys; rc = 0 if requests.get('http://localhost:${PUPPETBOARD_PORT}${PUPPETBOARD_URL_PREFIX:-}${PUPPETBOARD_STATUS_ENDPOINT}').ok else 255; sys.exit(rc)"
 
+RUN apk add --no-cache gcc libmemcached-dev libc-dev zlib-dev
 RUN mkdir -p /usr/src/app/
 WORKDIR /usr/src/app/
 COPY . /usr/src/app
 RUN pip install --no-cache-dir -r requirements-docker.txt .
 
-CMD gunicorn -b ${PUPPETBOARD_HOST}:${PUPPETBOARD_PORT} --workers="${PUPPETBOARD_WORKERS:-1}" -e SCRIPT_NAME="${PUPPETBOARD_URL_PREFIX:-}" --access-logfile=- puppetboard.app:app
+CMD gunicorn -b ${PUPPETBOARD_HOST}:${PUPPETBOARD_PORT} --preload --workers="${PUPPETBOARD_WORKERS:-1}" -e SCRIPT_NAME="${PUPPETBOARD_URL_PREFIX:-}" --access-logfile=- puppetboard.app:app

README.md

@@ -216,6 +216,22 @@ Other settings that might be interesting, in no particular order:
     in the last report. Otherwise shows only 'some' string if there are resources with given status. Setting this
     to `False` gives performance benefits, especially in big Puppet environments (more than few hundreds of nodes).
     Defaults to `True`.
+- `ENABLE_CLASS`: If set to `True` allows the user to view the number of resource events (number of changed resources in the last report) grouped by class.
+    The resource events are grouped by their status ('failure', 'success', 'noop'). 
+- `CLASS_EVENTS_STATUS_COLUMNS`: A mapping between the status of the resource events and the name of the columns of the table to display.
+- `CACHE_TYPE`: Specifies which type of caching object to use when `SCHEDULER_ENABLED` is set to `True`.
+    The cache is used for the classes view (`ENABLE_CLASS` is set to `True`) which requires parsing the events of all the latest reports to group them by Puppet class.
+    If the last report is present in the cache, we do not parse the events, which avoids unnecessary processing.
+    If you configure more than one worker, you must use a shared backend (e.g. `MemcachedCache`) to allow the sharing of the cache between the processes.
+    Indeed, the `SimpleCache` type does not allow sharing the cache between processes, it uses the process memory to store the cache.
+    Defaults to `SimpleCache`.
+- `CACHE_DEFAULT_TIMEOUT`: Cache lifetime in second. Defaults to `3600`.
+- `SCHEDULER_ENABLED`: If set to `True` then a scheduler instance is created in order to execute scheduled jobs. Defaults to `False`.
+- `SCHEDULER_JOBS`: List of the scheduled jobs to trigger within a worker.
+    A job can for example be used to compute a result to be cached. This is the case for the classes view which uses a job to pre-compute at regular intervals the results to be displayed. 
+    Each scheduled job must contain the following fields: `id`, `func`, `trigger`, `seconds`.
+- `SCHEDULER_LOCK_BIND_PORT`: Specifies an available port that allows a single worker to listen on it.
+    This allows to configure scheduled jobs in a single worker. Defaults to `49100`.
 
 ## Getting Help<a id="getting-help"></a>
 
@@ -346,6 +362,14 @@ by Tim Pope](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.ht
 
 ![Inventory view](https://raw.githubusercontent.com/voxpupuli/puppetboard/master/screenshots/inventory.png)
 
+* Classes view, group the resource events of the last reports by Puppet class 
+
+![Classes view](https://raw.githubusercontent.com/voxpupuli/puppetboard/master/screenshots/classes.png)
+
+* Class view, list the nodes with almost one resource event for a given class
+
+![Class view](https://raw.githubusercontent.com/voxpupuli/puppetboard/master/screenshots/class.png)
+
 # Legal<a id="legal"></a>
 
 The app code is licensed under the [Apache License, Version 2.0](./LICENSE).

puppetboard/app.py

@@ -12,6 +12,8 @@
 # noinspection PyUnresolvedReferences
 import puppetboard.views.catalogs  # noqa: F401
 # noinspection PyUnresolvedReferences
+import puppetboard.views.classes  # noqa: F401
+# noinspection PyUnresolvedReferences
 import puppetboard.views.dailychart  # noqa: F401
 # noinspection PyUnresolvedReferences
 import puppetboard.views.facts  # noqa: F401
@@ -33,12 +35,13 @@
 import puppetboard.views.failures  # noqa: F401
 import puppetboard.errors  # noqa: F401
 
-from puppetboard.core import get_app, get_puppetdb
+from puppetboard.core import get_app, get_puppetdb, get_scheduler
 from puppetboard.version import __version__
 from puppetboard.utils import is_a_test, check_db_version, check_secret_key
 
 app = get_app()
 puppetdb = get_puppetdb()
+get_scheduler()
 running_as = os.path.basename(sys.argv[0])
 if not is_a_test():
     check_db_version(puppetdb)
@@ -56,6 +59,7 @@
     ('metrics', 'Metrics'),
     ('inventory', 'Inventory'),
     ('catalogs', 'Catalogs'),
+    ('classes', 'Classes'),
     ('radiator', 'Radiator'),
     ('query', 'Query'),
 ]
@@ -66,6 +70,9 @@
 if not app.config.get('ENABLE_CATALOG'):
     menu_entries.remove(('catalogs', 'Catalogs'))
 
+if not app.config.get('ENABLE_CLASS'):
+    menu_entries.remove(('classes', 'Classes'))
+
 app.jinja_env.globals.update(menu_entries=menu_entries)
 
 

puppetboard/core.py

@@ -1,8 +1,11 @@
 import logging
 import re
+import socket
 
 import pkg_resources
 from flask import Flask
+from flask_caching import Cache
+from flask_apscheduler import APScheduler
 from pypuppetdb import connect
 
 from puppetboard.utils import (get_or_abort, jsonprint,
@@ -28,6 +31,8 @@
 
 APP = None
 PUPPETDB = None
+CACHE = None
+SCHEDULER = None
 
 
 def get_app():
@@ -46,6 +51,7 @@ def get_app():
         app.jinja_env.filters['jsonprint'] = jsonprint
         app.jinja_env.globals['url_for_field'] = url_for_field
         app.jinja_env.globals['quote_columns_data'] = quote_columns_data
+        app.jinja_env.add_extension('jinja2.ext.do')
         APP = app
 
     return APP
@@ -75,6 +81,44 @@ def get_puppetdb():
     return PUPPETDB
 
 
+def get_cache():
+    global CACHE
+
+    if CACHE is None:
+        app = get_app()
+        cache = Cache()
+        cache.init_app(app)
+
+        CACHE = cache
+
+    return CACHE
+
+
+def get_scheduler():
+    global SCHEDULER
+
+    if SCHEDULER is None:
+        app = get_app()
+        if not app.config['SCHEDULER_ENABLED']:
+            return SCHEDULER
+
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.bind(('127.0.0.1', app.config['SCHEDULER_LOCK_BIND_PORT']))
+        except socket.error:
+            print('scheduler already running: socket.error')
+            return SCHEDULER
+        else:
+            print('scheduler enabled')
+            scheduler = APScheduler()
+            scheduler.init_app(app)
+            scheduler.start()
+
+            SCHEDULER = scheduler
+
+    return SCHEDULER
+
+
 def environments() -> dict:
     envs = {}
     puppetdb = get_puppetdb()

puppetboard/default_settings.py

@@ -68,3 +68,32 @@
     'test',
     'dev',
 ]
+
+ENABLE_CLASS = False
+# mapping between the status of the events (from PuppetDB) and the columns of the table to display
+CLASS_EVENTS_STATUS_COLUMNS = [
+    # ('skipped', 'Skipped'),
+    ('failure', 'Failure'),
+    ('success', 'Success'),
+    ('noop', 'Noop'),
+]
+# Type of caching object to use when `SCHEDULER_ENABLED` is set to `True`.
+# If more than one worker, use a shared backend (e.g. `MemcachedCache`) 
+# to allow the sharing of the cache between the processes.
+CACHE_TYPE = 'SimpleCache'
+# Cache litefime in second
+CACHE_DEFAULT_TIMEOUT = 3600
+
+# List of scheduled jobs to trigger
+#   * `id`: job's ID
+#   * `func`: full path of the function to execute
+#   * `trigger`: should be 'interval' if you want to run the job at regular intervals
+#   * `seconds`: number of seconds between 2 triggered jobs
+SCHEDULER_JOBS = [{
+    'id': 'do_build_async_cache_1',
+    'func': 'puppetboard.schedulers.classes:build_async_cache',
+    'trigger': 'interval',
+    'seconds': 300,
+}]
+SCHEDULER_ENABLED = False
+SCHEDULER_LOCK_BIND_PORT = 49100

puppetboard/docker_settings.py

@@ -151,3 +151,49 @@ def coerce_bool(v, default):
     'dev',
 ])
 FAVORITE_ENVS = [x.strip() for x in os.getenv('FAVORITE_ENVS', FAVORITE_ENVS_DEF).split(',')]
+
+
+# Enable classes view (displays the number of changed resources
+# by status and class)
+ENABLE_CLASS = coerce_bool(os.getenv('ENABLE_CLASS'), False)
+
+# Use caching if classes view is enabled
+CACHE_DEFAULT_TIMEOUT = int(os.getenv('CACHE_DEFAULT_TIMEOUT', '3600'))
+CACHE_TYPE = os.getenv('CACHE_TYPE', 'SimpleCache')
+CACHE_MEMCACHED_SERVERS_DEFAULT = ','.join(['memcached:11211'])
+if CACHE_TYPE == 'MemcachedCache':
+    CACHE_MEMCACHED_SERVERS = os.getenv('CACHE_MEMCACHED_SERVERS', CACHE_MEMCACHED_SERVERS_DEFAULT).split(',')
+
+# A mapping between the status of the resource events
+# and the name of the columns of the table to display.
+CLASS_EVENTS_STATUS_COLUMNS_DEFAULT = ','.join(['failure', 'Failure',
+                                                # 'skipped', 'Skipped',
+                                                'success', 'Success',
+                                                'noop', 'Noop'])
+
+CLASS_EVENTS_STATUS_COLUMNS_STR = os.getenv('CLASS_EVENTS_STATUS_COLUMNS', CLASS_EVENTS_STATUS_COLUMNS_DEFAULT).split(',')
+
+# Take the Array and convert it to a tuple
+CLASS_EVENTS_STATUS_COLUMNS = [(CLASS_EVENTS_STATUS_COLUMNS_STR[i].strip(),
+                                CLASS_EVENTS_STATUS_COLUMNS_STR[i + 1].strip()) for i in range(0, len(CLASS_EVENTS_STATUS_COLUMNS_STR), 2)]
+
+# Enabled a scheduler instance to trigger jobs in background. 
+SCHEDULER_ENABLED = coerce_bool(os.getenv('SCHEDULER_ENABLED'), False)
+
+# Tuples are hard to express as an environment variable, so here
+# the tuple can be listed as a list of items
+# Examples:
+#   export SCHEDULER_JOBS="id, <id>, func, <func>, trigger, <trigger>, seconds, <seconds>"
+# The scheduled jobs are separated by the character ";".
+SCHEDULER_JOBS_DEFAULT = ';'.join([','.join(['id', 'do_build_async_cache_1',
+                                   'func', 'puppetboard.schedulers.classes:build_async_cache',
+                                   'trigger', 'interval',
+                                   'seconds', '300'])])
+
+SCHEDULER_JOBS_STR = os.getenv('SCHEDULER_JOBS', SCHEDULER_JOBS_DEFAULT).split(';')
+
+# Take the Array and convert it to a tuple
+SCHEDULER_JOBS = []
+for job in SCHEDULER_JOBS_STR:
+    SCHEDULER_JOBS.append({job.split(',')[i]: (int(job.split(',')[i + 1]) if job.split(',')[i] == 'seconds' else job.split(',')[i + 1])
+                           for i in range(0, len(job.split(',')), 2)})

puppetboard/schedulers/classes.py

@@ -0,0 +1,61 @@
+from pypuppetdb.QueryBuilder import (AndOperator, InOperator, FromOperator,
+                                     EqualsOperator, NullOperator, OrOperator,
+                                     ExtractOperator, LessEqualOperator, SubqueryOperator)
+
+from puppetboard.core import get_app, get_cache, get_puppetdb, environments, stream_template, REPORTS_COLUMNS
+from puppetboard.utils import (yield_or_stop, check_env, get_or_abort)
+from puppetboard.views.classes import (get_status_from_events)
+
+app = get_app()
+cache = get_cache()
+puppetdb = get_puppetdb()
+
+events_status_columns = ['skipped','failure','success','noop']
+
+
+def build_async_cache():
+    """Scheduled job triggered at regular interval in order to pre-compute the
+    results to display in the classes view.
+    The result contains the events associated with the last reports and is
+    stored in the cache.
+    """
+    columns = [col for col in app.config['CLASS_EVENTS_STATUS_COLUMNS'] if col[0] in events_status_columns]
+
+    envs = puppetdb.environments()
+    for env in puppetdb.environments():
+        env = env['name']
+        query = AndOperator()
+        query.add(EqualsOperator("environment", env))
+        # get events from last report for each active node
+        query_in = InOperator('hash')
+        query_ex = ExtractOperator()
+        query_ex.add_field('latest_report_hash')
+        query_from = FromOperator('nodes')
+        query_null = NullOperator('deactivated', True)
+        query_ex.add_query(query_null)
+        query_from.add_query(query_ex)
+        query_in.add_query(query_from)
+        reportlist = puppetdb.reports(query=query_in)
+
+        new_cache = {}
+        for report in yield_or_stop(reportlist):
+            report_hash = report.hash_
+            for event in yield_or_stop(report.events()):
+                containing_class = event.item['class']
+                status = event.status
+                new_cache[containing_class] = new_cache.get(containing_class, {})
+                new_cache[containing_class][report_hash] = new_cache[containing_class].get(report_hash, {
+                    'node_name': report.node,
+                    'node_status': report.status,
+                    'class_status': 'skipped',
+                    'report_hash': report_hash,
+                    'nb_events_per_status': {col[0]: 0 for col in columns},
+                })
+                if status in new_cache[containing_class][report_hash]['nb_events_per_status']:
+                    new_cache[containing_class][report_hash]['nb_events_per_status'][status] += 1
+        for class_name in new_cache:
+            for report_hash, report in new_cache[class_name].items():
+                status = get_status_from_events(report['nb_events_per_status'])
+                new_cache[class_name][report_hash]['class_status'] = get_status_from_events(report['nb_events_per_status'])
+
+        cache.set(f'classes_resource_{env}', new_cache)

puppetboard/static/css/puppetboard.css

@@ -60,15 +60,15 @@ h1.ui.header.no-margin-bottom {
   color: #AA4643;
 }
 
-.ui.label.failed, .ui.label.events.failure {
+.ui.label.failed, .ui.label.failure, .ui.label.events.failure {
   background-color: #AA4643;
 }
 
 .ui.header.changed, .ui.line.changed {
   color: #4572A7;
 }
 
-.ui.label.changed, .ui.label.events.success {
+.ui.label.changed, .ui.label.success, .ui.label.events.success {
   background-color: #4572A7;
 }
 
@@ -88,7 +88,7 @@ h1.ui.header.no-margin-bottom {
   background-color: #DB843D;
 }
 
-.ui.label.resources.total {
+.ui.label.resources.total, .ui.label.nodes.total {
   background-color: #989898;
 }
 

puppetboard/templates/_macros.html

@@ -10,6 +10,20 @@
   {% endif %}
 {%- endmacro %}
 
+{% macro event_status_counts(caller, status, count) -%}
+  {% if count == 0 %}
+    <span class="ui small count label">0</span>
+  {% elif status == 'failure' %}
+    <span class="ui small count label failed" title="events failure">{{count}}</span>
+  {% elif status == 'success' %}
+    <span class="ui small count label changed" title="events success">{{count}}</span>
+  {% elif status == 'skipped' %}
+    <span class="ui small count label skipped" title="events skipped">{{count}}</span>
+  {% elif status == 'noop' %}
+    <span class="ui small count label noop" title="events noop">{{count}}</span>
+  {% endif %}
+{%- endmacro %}
+
 {% macro report_status(caller, status, node_name, metrics, current_env, unreported_time=False, report_hash=False) -%}
   <a class="ui {{status}} label status" href="{{url_for('report', env=current_env, node_name=node_name, report_id=report_hash)}}">{{ status|upper }}</a>
   {% if status == 'unreported' %}

puppetboard/templates/class.html

@@ -0,0 +1,22 @@
+{% extends 'layout.html' %}
+{% import '_macros.html' as macros %}
+{% block content %}
+<h1 id="class_name">{{ class_name }}</h1>
+<table id="class_table" class='ui fixed compact very basic sortable table'>
+  <thead>
+    <tr>
+      <th>Node</th>
+      <th>Node Status</th>
+      <th>Class Status</th>
+    </tr>
+  </thead>
+  <tbody class="searchable">
+  </tbody>
+</table>
+{% endblock content %}
+{% block onload_script %}
+{% macro extra_options(caller) %}
+  'serverSide': false,
+{% endmacro %}
+{{ macros.datatable_init(table_html_id="class_table", ajax_url=url_for('class_resource_ajax', env=current_env, class_name=class_name), data=None, default_length=config.NORMAL_TABLE_COUNT, length_selector=config.TABLE_COUNT_SELECTOR, extra_options=extra_options) }}
+{% endblock onload_script %}

puppetboard/templates/class_resource.json.tpl

@@ -0,0 +1,16 @@
+{%- import '_macros.html' as macros -%}
+{
+  "draw": {{draw}},
+  "recordsTotal": {{total}},
+  "recordsFiltered": {{total_filtered}},
+  "data": [
+    {% for report_hash, node in nodes_data.items() -%}
+      {%- if not loop.first %},{%- endif -%}
+      [
+        {% filter jsonprint %}<a href="{{ url_for('node', env=current_env, node_name=node.node_name) }}">{{ node.node_name }}</a>{% endfilter %},
+        {% filter jsonprint %}<a class="ui {{ node.node_status }} label status" href="{{url_for('report', env=current_env, node_name=node.node_name, report_id=node.report_hash)}}">{{ node.node_status|upper }}</a>{% endfilter %},
+        {% filter jsonprint %}<a class="ui {{ node.class_status }} label status" href="{{url_for('report', env=current_env, node_name=node.node_name, report_id=node.report_hash)}}#events">{{ node.class_status|upper }}</a>{% endfilter %}
+      ]
+    {% endfor -%}
+  ]
+}