Add first-class handling for AlertManager webhook types with links from annotations

[Arkaniad]

We're using the following JS transformation for creating Generic Webhooks for Alertmanager alerting to Matrix rooms (and I'm not entirely sure where we got it from at the moment )

function statusBadge(status, severity) {
    let statusColor;
    if (status === "resolved") {
        return `<font color='green'><b>[RESOLVED]</b></font>`;
    }

    switch(severity) {
        case 'resolved':
        case 'critical':
            return `<font color='red'><b>[FIRING - CRITICAL]</b></font>`;
        case 'warning':
            return `<font color='orange'><b>[FIRING - WARNING]</b></font>`;
        default:
            return `<b>[${status.toUpperCase()}]</b>`;
    }
}

function silenceLink(alert, externalURL) {
    filters = []
    for (const [label, val] of Object.entries(alert.labels)) {
        filters.push(encodeURIComponent(`${label}="${val}"`));
    }
    return `<a href="${externalURL}#silences/new?filter={${filters.join(",")}}">silence</a>`;
}

if (!data.alerts) { 
    result = {
        version: 'v2',
        empty: true,
    };
    return;
}

const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;

for (const alert of data.alerts) {
    plainErrors.push(`**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.foo.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} [source](${alert.generatorURL})`);
    htmlErrors.push(`${statusBadge(alert.status, alert.labels.severity)} <a href="https://wiki.foo.dev/alerts/${alert.labels.alertname}">${alert.labels.alertname}</a>: ${alert.annotations.description} <a href="${alert.generatorURL}">source</a> ${silenceLink(alert, externalURL)}`)
    result = {
        version: 'v2',
        plain: plainErrors.join(`\n\n`),
        html: htmlErrors.join(`<br/>`),
        msgtype: 'm.text'
    };
}

This works well enough, but it gets complicated now that we're exploring appending further context URLs (grafana links, ArgoCD links etc) inside the alert's annotations field like so:

  groups:
  - name: argocd-apps
    rules:
    - alert: ArgoCDAppStuckUnhealthy
      annotations:
        runbook_url: https://wiki.foo.dev/argocd/ArgoCDAppStuckUnhealthy
        summary: ArgoCD Application is in unhealthy state
        description: |-
          ArgoCD Application '{{ $labels.name }}' in namespace {{ $labels.dest_namespace }} is in {{ $labels.health_status }} state for longer than 5 minutes.
        application_argocd_url: "https://argocd.foo.dev/applications/{{ $labels.name }}"
      expr: |-
        sum(argocd_app_info{ health_status!="Healthy", dest_namespace=~"{{ $targetNamespace }}" }) by (name, dest_namespace, health_status) > 0
      for: 5m
      labels:
        severity: warning

At first I tried stuffing the URL into the description but the Javascript hook we've ended up with didn't seem to parse it correctly into a link (whether I used markdown or html, could be pebcak issue), but it would be cool to have the ability to specify an AlertManager webhook without a transform and have a map of annotation fields and link labels that gets built into a list of links in the issue, like this:

Which we've clumsily hacked together with this transform:

function statusBadge(status, severity) {
  let statusColor;
  if (status === "resolved") {
    return `<font color='green'><b>[RESOLVED]</b></font>`;
  }

  switch (severity) {
    case 'resolved':
    case 'critical':
      return `<font color='red'><b>[FIRING - CRITICAL]</b></font>`;
    case 'warning':
      return `<font color='orange'><b>[FIRING - WARNING]</b></font>`;
    default:
      return `<b>[${status.toUpperCase()}]</b>`;
  }
}

function wrapUrlHTML(link) {
  return `<a href="${link.href}">${link.text}</a>`
}

function wrapLinkListHTML(links) {
  output = "<ul>\n"
  for (i = 0; i < links.length; i++) {
    output += `<li>${wrapUrlHTML(links[i])}</li>\n`
  }
  output += "</ul>\n"

  return output
}

function wrapLinkListMD(links) {
  output = "";
  for (i = 0; i < links.length; i++) {
    output += `- ${wrapUrlHTML(links[i])}</li>\n`
  }

  return output
}

function silenceLink(alert, externalURL) {
  filters = []
  for (const [label, val] of Object.entries(alert.labels)) {
    filters.push(encodeURIComponent(`${label}="${val}"`));
  }

  return {
    href: `${externalURL}#silences/new?filter={${filters.join(", ")}}`,
    text: "Silence alert"
  }
}

function alertLinks(alert, externalURL) {
  links = []

  appName = `${alert.labels.pod}/${alert.labels.container}`

  links.push(sourceLink(alert));
  links.push(silenceLink(alert, externalURL));

  if (alert.annotations.hasOwnProperty('application_argocd_url')) {
    links.push({
      href: alert.annotations.application_argocd_url,
      text: `View App in ArgoCD`
    });
  }

  if (alert.annotations.hasOwnProperty('logs_url')) {
    links.push({
      href: alert.annotations.logs_url,
      text: `View logs for ${appName}`
    });
  }

  if (alert.annotations.hasOwnProperty('argocd_logs_url')) {
    links.push({
      href: alert.annotations.argocd_logs_url,
      text: `View logs for ArgoCD`
    });
  }
  return links;
}

function sourceLink(alert) {
  return {
    href: alert.generatorURL,
    text: "View alert in VMAlert"
  }
}

function alertWikiLink(alert) {
  return {
    href: `https://wiki.element.dev/alerts/${alert.labels.alertname}`,
    text: alert.labels.alertname
  }
}

function buildAlertHtml(alert, externalURL) {
  return `${statusBadge(alert.status, alert.labels.severity)} ${wrapUrlHTML(alertWikiLink(alert))}: ${alert.annotations.description}\n${wrapLinkListHTML(alertLinks(alert, externalURL))}`
}

function buildAlertPlain(alert, externalURL) {
  return `**[${alert.status.toUpperCase()} - ${alert.labels.severity}]** - [${alert.labels.alertname}](https://wiki.element.dev/alerts/${alert.labels.alertname}): ${alert.annotations.description} \n${wrapLinkListMD(alertLinks(alert, externalURL))}`
}

if (!data.alerts) {
  result = {
    version: 'v2',
    empty: true,
  };
  return;
}

const plainErrors = [];
const htmlErrors = [];
const { externalURL, alerts } = data;

for (const alert of data.alerts) {
  plainErrors.push(buildAlertPlain(alert, externalURL));
  htmlErrors.push(buildAlertHtml(alert, externalURL));
  result = {
    version: 'v2',
    plain: plainErrors.join(`\n\n`),
    html: htmlErrors.join(`<br/>`),
    msgtype: 'm.text'
  };
}

This seems overkill, and won't scale for us, especially since this is a manual configuration step and can't be pre-provisioned / managed via IaC as far as I can tell.

Would it be possible to create a class of webhook aside from Generic Webhook that could handle Alertmanager hooks with it's own configuration to escape some of this transform stuff?

[wrenix]

Does it update/edit the message from firing to resolved?

Current i miss a table of the alert labels, what we get with another tool (but that should be easy to add).

It would be nice, when this tool get alertmanager support per default

[jessebot]

This is actually the number one use case I want hookshot for, so if Alert Manager had first class support, I'd work harder on helping with the helm chart here too. 🙏

_{Also, hi @wrenix, you are everywhere I am 😁}