From 5255e2dd6e3b56afd455e49ee0e154df82b8234f Mon Sep 17 00:00:00 2001
From: Thomas Heartman <thomas@getunleash.io>
Date: Wed, 10 Apr 2024 08:49:15 +0200
Subject: [PATCH] chore: Establish a baseline for the number of envs disabled
 per project (#6807)

This PR adds a counter in Prometheus for counting the number of
"environment disabled" events we get per project. The purpose of this is
to establish a baseline for one of the "project management UI" project's
key results.

## On gauges vs counters

This PR uses a counter. Using a gauge would give you the total number of
envs disabled, not the number of disable events. The difference is
subtle, but important.

For projects that were created before the new feature, the gauge might
be appropriate. Because each disabled env would require at least one
disabled event, we can get a floor of how many events were triggered for
each project.

However, for projects created after we introduce the planned change,
we're not interested in the total envs anymore, because you can disable
a hundred envs on creation with a single action. In this case, a gauge
showing 100 disabled envs would be misleading, because it didn't take
100 events to disable them.

So the interesting metric here is how many times did you specifically
disable an environment in project settings, hence the counter.

## Assumptions and future plans

To make this easier on ourselves, we make the follow assumption: people
primarily disable envs **when creating a project**.

This means that there might be a few lagging indicators granting some
projects a smaller number of events than expected, but we may be able to
filter those out.

Further, if we had a metric for each project and its creation date, we
could correlate that with the metrics to answer the question "how many
envs do people disable in the first week? Two weeks? A month?". Or
worded differently: after creating a project, how long does it take for
people to configure environments?

Similarly, if we gather that data, it will also make filtering out the
number of events for projects created **after** the new changes have
been released much easier.

The good news: Because the project creation metric with dates is a
static aggregate, it can be applied at any time, even retroactively, to
see the effects.
---
 src/lib/metrics.test.ts | 17 +++++++++++++++++
 src/lib/metrics.ts      | 10 ++++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/lib/metrics.test.ts b/src/lib/metrics.test.ts
index be2574d228bf..f1be259d33ff 100644
--- a/src/lib/metrics.test.ts
+++ b/src/lib/metrics.test.ts
@@ -8,6 +8,7 @@ import {
     CLIENT_REGISTER,
     FEATURE_ENVIRONMENT_ENABLED,
     FEATURE_UPDATED,
+    PROJECT_ENVIRONMENT_REMOVED,
 } from './types/events';
 import { createMetricsMonitor } from './metrics';
 import createStores from '../test/fixtures/store';
@@ -258,3 +259,19 @@ test('Should not collect client sdk version if sdkVersion is of wrong format or
     );
     expect(metrics).not.toMatch(/unleash-client-rust/);
 });
+
+test('should collect metrics for project disabled numbers', async () => {
+    eventStore.emit(PROJECT_ENVIRONMENT_REMOVED, {
+        project: 'default',
+        environment: 'staging',
+        createdBy: 'Jay',
+        createdByUserId: 26,
+    });
+
+    const recordedMetric = await prometheusRegister.getSingleMetricAsString(
+        'project_environments_disabled',
+    );
+    expect(recordedMetric).toMatch(
+        /project_environments_disabled{project_id=\"default\"} 1/,
+    );
+});
diff --git a/src/lib/metrics.ts b/src/lib/metrics.ts
index 33f025a97f2f..186a18caa876 100644
--- a/src/lib/metrics.ts
+++ b/src/lib/metrics.ts
@@ -18,6 +18,7 @@ import {
     FEATURE_UPDATED,
     CLIENT_METRICS,
     CLIENT_REGISTER,
+    PROJECT_ENVIRONMENT_REMOVED,
 } from './types/events';
 import type { IUnleashConfig } from './types/option';
 import type { IUnleashStores } from './types/stores';
@@ -258,6 +259,12 @@ export default class MetricsMonitor {
             help: 'Duration of mapFeaturesForClient function',
         });
 
+        const projectEnvironmentsDisabled = createCounter({
+            name: 'project_environments_disabled',
+            help: 'How many "environment disabled" events we have received for each project',
+            labelNames: ['project_id'],
+        });
+
         async function collectStaticCounters() {
             try {
                 const stats = await instanceStatsService.getStats();
@@ -607,6 +614,9 @@ export default class MetricsMonitor {
                 });
             }
         });
+        eventStore.on(PROJECT_ENVIRONMENT_REMOVED, ({ project }) => {
+            projectEnvironmentsDisabled.increment({ project_id: project });
+        });
 
         await this.configureDbMetrics(db, eventBus, schedulerService);