Monitor System Upgrades with Healthchecks.io

kusold · Jan 19, 2024 · f3ae953 · f3ae953
1 parent ab31f85
commit f3ae953
Show file tree

Hide file tree

Showing 9 changed files with 62 additions and 0 deletions.
diff --git a/hosts/mallard/configuration.nix b/hosts/mallard/configuration.nix
@@ -12,6 +12,7 @@
       ../../modules/nix.nix
       ../../modules/base-pkgs.nix
       ../../modules/autoupgrade.nix
+      ../../modules/healthchecks-io-monitor.nix
       ../../modules/time-utc.nix
       ../../modules/tailscale.nix
 

diff --git a/hosts/mikes-desktop/configuration.nix b/hosts/mikes-desktop/configuration.nix
@@ -13,6 +13,7 @@
       ../../modules/nix.nix
       ../../modules/base-pkgs.nix
       ../../modules/autoupgrade.nix
+      ../../modules/healthchecks-io-monitor.nix
       ../../modules/time-denver.nix
       ../../modules/tailscale.nix
     ];

diff --git a/hosts/nix/configuration.nix b/hosts/nix/configuration.nix
@@ -8,6 +8,7 @@
       ../../modules/nix.nix
       ../../modules/base-pkgs.nix
       ../../modules/autoupgrade.nix
+      ../../modules/healthchecks-io-monitor.nix
       ../../modules/time-utc.nix
     ];
 

diff --git a/hosts/scannerpi/configuration.nix b/hosts/scannerpi/configuration.nix
@@ -9,6 +9,7 @@
       ./hardware-configuration.nix
       ../../modules/nix.nix
       ../../modules/autoupgrade.nix
+      ../../modules/healthchecks-io-monitor.nix
 
       ../../modules/scanservjs.nix
       ../../modules/tailscale.nix

diff --git a/hosts/yuki/configuration.nix b/hosts/yuki/configuration.nix
@@ -14,6 +14,7 @@
       ../../modules/base-pkgs.nix
       ../../modules/time-denver.nix
       ../../modules/autoupgrade.nix
+      ../../modules/healthchecks-io-monitor.nix
     ];
 
   # Bootloader.

diff --git a/modules/autoupgrade.nix b/modules/autoupgrade.nix
@@ -8,4 +8,10 @@
   system.autoUpgrade.flake = "github:kusold/nix-config";
   system.autoUpgrade.dates = "daily";
   system.autoUpgrade.persistent = true;
+
+  systemctl.service."nixos-upgrade" = {
+    onFailure = [ "healthchecks-monitor@nixos-upgrade-${config.networking.hostName}:failure" ]
+    onSuccess = [ "healthchecks-monitor@nixos-upgrade-${config.networking.hostName}:success" ]
+    wants = [ "multi-user.target" "healthchecks-monitor@nixos-upgrade-${config.networking.hostName}:start" ];
+  }
 }
diff --git a/modules/healthchecks-io-monitor.nix b/modules/healthchecks-io-monitor.nix
@@ -0,0 +1,35 @@
+
+{ config, pkgs, ... }:
+{
+    age.secrets.monitoring-healthchecks-url.file = ../secrets/monitoring-healthchecks-url.age;
+
+
+# [Unit]
+# Description=Important service
+# OnFailure=healthchecks-monitor@deda567a-21e0-4744-ba9e-603c51e258b0:failure.service
+# OnSuccess=healthcheck-monitor@deda567a-21e0-4744-ba9e-603c51e258b0:success.service
+# Wants=healthcheck-monitor@deda567a-21e0-4744-ba9e-603c51e258b0:start.service
+
+  systemd.services.healthchecks-monitor = {
+    description = "Pings healthchecks (%i)";
+    wantedBy = [ "multi-user.target" ];
+    path = [ pkgs.curl ];
+    scriptArgs = "%i";
+    script = ''
+      HC_URL=$(cat ${age.secrets.monitoring-healthchecks-url.file})
+      IFS=: read -r SLUG ACTION <<< "%i"
+      if [ "$ACTION" = "start" ]; then
+        LOGS=""
+        EXIT_CODE="start"
+      else
+        LOGS=$(journalctl --no-pager -n 50 -u $MONITOR_UNIT)
+        EXIT_CODE=$MONITOR_EXIT_STATUS
+      fi
+      curl -fSs -m 10 --retry 3 --data-raw "$LOGS" "$HC_URL/$SLUG/$EXIT_CODE?create=1"
+    '';
+    serviceConfig = {
+      Type = "oneshot";
+    };
+  };
+
+}
diff --git a/secrets/monitoring-healthchecks-url.age b/secrets/monitoring-healthchecks-url.age
@@ -0,0 +1,15 @@
+age-encryption.org/v1
+-> ssh-ed25519 wif8hg y9pSpLSRoSL33nMjYmiQXHJoF6iNaHwmy9yYv7083HA
+3f2i9v701+Anszp08IKzDJ7FMLJynNa7CZaUE1U86fU
+-> ssh-ed25519 UoFiFw XEQeUqkKDKlmq62EFKGuUJHdW2Joe/VIeZYZSfszLDg
+2g4e+iYXjq+MwdvGr5zD6q+bsWCQfPpZdzDJoNTqngs
+-> ssh-ed25519 iBa5uA NuUybgtIFuid8kX1SrzV9oc6cfZqbPB9yZ1CFNUXpnA
++pIUsAi6dlCKUs6muzh1indPpCuvAdGWSp+5G+0UdCs
+-> ssh-ed25519 zwzZhA ciDHJxI/j+Pv4+mLdZTGKhgu5xkBX/wNxwjWSyB5fEo
+sQOj9R0GYmQ7leKK2YR8YKHLM9qKWU1mMjRN9cXw1Og
+-> ssh-ed25519 32ekKQ SEPfPSNhe0g97C2XecG9yOWFpjBAkbzf50M+r3sojkE
+X5GnBPqzFsEOSvX5XVsmKkj4PkWncE6M8CsVQ1wzCYw
+-> ssh-ed25519 ogl6Cg Vos8gCiIMfnby8nQYwbabbOl952zJpInv9O3ma48eG4
+ZXhR+69Gxc+7eoQbc++wraTTQB7Iz+UrRtlM+REwm1M
+--- UvVdQOF5XJaSv6SqmxXiErVEZUh1Uc96J+yWItlibxM
+�_��z�M����n����,��z�m�ԪƢ�*Rz5�|p[5���`�����<�7�Ξ\zy%�����
diff --git a/secrets/secrets.nix b/secrets/secrets.nix
@@ -20,4 +20,5 @@ in
   "app-jellyplex-watched.age".publicKeys = users ++ [mallard];
   "user-mike-hashed-passwd.age".publicKeys = users ++ systems;
   "github-access-token.age".publicKeys = users ++ systems; # Expires 2025-01-18
+  "monitoring-healthchecks-url.age".publicKeys = users ++ systems;
 }