utils.yml

# tasks that let you interact with the indexer for operational requirements
version: '3'

tasks:
  get-recent-oom:
    desc: Get recent OOM events from the cluster
    cmds:
    - |
      set - {{ .CLI_ARGS }}
      set -euo pipefail

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Fetching pod data from the cluster..."
      PODS_JSON=$(kubectl get pods --all-namespaces -o json)

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Processing OOMKilled pod information..."

      # Use jq to:
      # - Extract pods that have OOMKilled containers.
      # - For each pod, gather OOMKilled containers and their timestamps.
      # - Gather other containers in the same pod that are currently running.
      PODS_OOM=$(echo "$PODS_JSON" | jq '[.items[]
        | {
            name: .metadata.name,
            namespace: .metadata.namespace,
            nodeName: .spec.nodeName,

            # OOMKilled containers
            oomContainers: [
              .status.containerStatuses[]?
              | select(.lastState.terminated?.reason == "OOMKilled")
              | {
                  containerName: .name,
                  reason: .lastState.terminated.reason,
                  finishedAt: .lastState.terminated.finishedAt
                }
            ],

            # Other currently running containers in the same pod
            otherContainers: [
              .status.containerStatuses[]?
              | select((.lastState.terminated?.reason != "OOMKilled") and (.state.running?))
              | {
                  containerName: .name,
                  startedAt: .state.running.startedAt
                }
            ],

            # Container specs for reference
            containerSpecs: [
              .spec.containers[]
              | {
                  containerName: .name,
                  memoryRequest: (.resources.requests.memory // "None"),
                  memoryLimit: (.resources.limits.memory // "None")
                }
            ]
          }
        | select((.oomContainers | length) > 0)
      ] | sort_by(.oomContainers[].finishedAt)')

      COUNT=$(echo "$PODS_OOM" | jq 'length')

      echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] Found $COUNT pods with OOMKilled containers."

      if [ "$COUNT" -eq 0 ]; then
          echo "No pods found with OOMKilled containers."
          exit 0
      fi

      echo ""
      echo "========== OOMKilled Pods Detailed Report =========="
      echo ""

      # JQ filter to add relative hours
      # Given a timestamp in RFC3339 format (e.g. "2024-12-11T11:21:39Z"),
      # calculate how many hours ago it was from now.
      jq_filter='
        def hours_ago($t):
          if $t == null then
            "N/A"
          else
            ($t | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) as $ftime
            | (now - $ftime) as $diff
            | if $diff < 0 then
                ($t + " (in the future?)")
              else
                ($t + " (" + ( ($diff / 3600) | floor | tostring ) + " hours ago)")
              end
          end;

        .[] 
        | (
            "Pod Name: " + .name 
            + "\nNamespace: " + .namespace 
            + "\nNode: " + (.nodeName // "N/A")
          )
          + "\n\nContainers OOMKilled:" 
          + (
            if (.oomContainers | length) > 0 then
              (.oomContainers
              | map(
                  "\n  - Container: " + .containerName 
                  + "\n    Reason: " + (.reason // "N/A")
                  + "\n    FinishedAt: " + (hours_ago(.finishedAt))
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n\nOther Running Containers (same pod):"
          + (
            if (.otherContainers | length) > 0 then
              (.otherContainers
              | map(
                  "\n  - Container: " + .containerName
                  + "\n    StartedAt: " + (.startedAt // "N/A")
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n\nContainer Specs (Memory):"
          + (
            if (.containerSpecs | length) > 0 then
              (.containerSpecs
              | map(
                  "\n  - Container: " + .containerName
                  + "\n    Request: " + .memoryRequest
                  + "\n    Limit: " + .memoryLimit
                )
              | join("")
              )
            else
              "\n  None"
            end
          )
          + "\n------------------------------------------\n"
      '

      echo "$PODS_OOM" | jq -r "$jq_filter"


  seal-secret:
    desc: "Util to encrypt secret values and create in cluster only k8s secrets"
    cmds:
      - |
        set - {{ .CLI_ARGS }}
        set -eu

        usage() {
          echo "Usage: seal-secret -- < -n namespace> < -s secret-name> < -k key> < -v value> [ -c context-name]"
          echo "< -n namespace> is the k8s namespace intended for the secret"
          echo "< -s secret-name> is the k8s secret name the controller will create"
          echo "< -k key> is the k8s secret key name"
          echo "< -v value> is the secret value being encrypted"
          echo "[-c context-name] will default to current-context unless provided"
          exit 1
        }

        NAMESPACE="${NAMESPACE:-}"
        SECRET_NAME="${NAME:-}"
        KEY="${KEY:-}"
        VALUE="${VALUE:-}"
        CLUSTER_CONTEXT="${CLUSTER_CONTEXT:-}"

        while getopts 'n:s:k:v:c:' opt; do
          case "$opt" in
            n) NAMESPACE="$OPTARG";;
            s) SECRET_NAME="$OPTARG";;
            k) KEY="$OPTARG";;
            v) VALUE="$OPTARG";;
            c) CLUSTER_CONTEXT="$OPTARG";;
            *) usage
              exit 1 ;;
          esac
        done

        if [ -z "$CLUSTER_CONTEXT" ]; then
            echo "Cluster context not set, using currently set context $(kubectl config current-context)"
            echo "If you're creating a secret for a different cluster rerun the script and pass the context name for the cluster"
            echo ""
            CLUSTER_CONTEXT=$(kubectl config current-context)
        else 
            echo "Context name set - creating secret $SECRET_NAME against $CLUSTER_CONTEXT"
            kubectl config use-context "$CLUSTER_CONTEXT"
        fi

        if [ -z "$VALUE" ]; then
          usage
        fi

        secretManifest=$(echo -n "${VALUE}" | kubectl create secret generic "${SECRET_NAME}" --dry-run=client --namespace "${NAMESPACE}"  --from-literal="${KEY}"="${VALUE}" -o json 2>&1)
        sealedManifest=`echo $secretManifest | kubeseal --namespace "${NAMESPACE}" --controller-namespace sealed-secrets --controller-name sealed-secrets  2>&1`
        sealedValue=`echo $sealedManifest | jq --arg 'key' "${KEY}" '.spec.encryptedData[$key]' -r`

        echo "${KEY}: '${sealedValue}'"

  octant:
    desc: Run octant to navigate your cluster in your web browser
    cmds:
    - octant {{.CLI_ARGS}}

  k9s:
    desc: Run k9s to navigate your cluster in your terminal
    cmds:
    - k9s {{.CLI_ARGS}}