feature: send errors to the pod's event to increase visibility

WARNING: starting with this commit, the Agent must be know about the
namespace in which the agent's pod is running:

- If the pod has a service account mounted, then the agent will load the
  namespace from the file
  /var/run/secrets/kubernetes.io/serviceaccount/namespace.
- Otherwise, you can provide the namespace using --install-namespace
  (meant for running the agent out-of-cluster for testing purposes).

Previously, --install-namespace was only needed when using the
VenafiConnection mode (i.e., using --venafi-connection).

deploy/charts/venafi-kubernetes-agent/templates/rbac.yaml

@@ -1,5 +1,31 @@
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "venafi-kubernetes-agent.fullname" . }}-event-emitted
+  labels:
+    {{- include "venafi-kubernetes-agent.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "venafi-kubernetes-agent.fullname" . }}-event-emitted
+  labels:
+    {{- include "venafi-kubernetes-agent.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "venafi-kubernetes-agent.fullname" . }}-event-emitted
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "venafi-kubernetes-agent.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: {{ include "venafi-kubernetes-agent.fullname" . }}-cluster-viewer

pkg/agent/config.go

@@ -273,8 +273,7 @@ func InitAgentCmdFlags(c *cobra.Command, cfg *AgentCmdFlags) {
 		"install-namespace",
 		"",
 		"For testing purposes. Namespace in which the agent is running. "+
-			"Only needed with the "+string(VenafiCloudVenafiConnection)+" mode"+
-			"when running the agent outside of Kubernetes.",
+			"Only needed when running the agent outside of Kubernetes.",
 	)
 	c.PersistentFlags().BoolVarP(
 		&cfg.Profiling,
@@ -314,6 +313,7 @@ type CombinedConfig struct {
 	BackoffMaxTime time.Duration
 	StrictMode     bool
 	OneShot        bool
+	InstallNS      string
 
 	// Used by JetstackSecureOAuth, JetstackSecureAPIToken, and
 	// VenafiCloudKeypair. Ignored in VenafiCloudVenafiConnection mode.
@@ -330,7 +330,6 @@ type CombinedConfig struct {
 	// VenafiCloudVenafiConnection mode only.
 	VenConnName string
 	VenConnNS   string
-	InstallNS   string
 
 	// Only used for testing purposes.
 	OutputPath string
@@ -530,20 +529,20 @@ func ValidateAndCombineConfig(log *log.Logger, cfg Config, flags AgentCmdFlags)
 		res.StrictMode = flags.StrictMode
 	}
 
-	// Validation of --venafi-connection, --venafi-connection-namespace, and
-	// --install-namespace.
-	if res.AuthMode == VenafiCloudVenafiConnection {
-		var installNS string = flags.InstallNS
-		if flags.InstallNS == "" {
-			var err error
-			installNS, err = getInClusterNamespace()
-			if err != nil {
-				errs = multierror.Append(errs, fmt.Errorf("could not guess which namespace the agent is running in: %w", err))
-			}
+	// Validation of --install-namespace.
+	var installNS string = flags.InstallNS
+	if flags.InstallNS == "" {
+		var err error
+		installNS, err = getInClusterNamespace()
+		if err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("could not guess which namespace the agent is running in: %w", err))
 		}
-		res.InstallNS = installNS
-		res.VenConnName = flags.VenConnName
+	}
+	res.InstallNS = installNS
 
+	// Validation of --venafi-connection and --venafi-connection-namespace.
+	if res.AuthMode == VenafiCloudVenafiConnection {
+		res.VenConnName = flags.VenConnName
 		var venConnNS string = flags.VenConnNS
 		if flags.VenConnNS == "" {
 			venConnNS = installNS

pkg/agent/run.go

@@ -18,11 +18,18 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientgocorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/jetstack/preflight/api"
 	"github.com/jetstack/preflight/pkg/client"
 	"github.com/jetstack/preflight/pkg/datagatherer"
+	"github.com/jetstack/preflight/pkg/kubeconfig"
 	"github.com/jetstack/preflight/pkg/logs"
 	"github.com/jetstack/preflight/pkg/version"
 
@@ -115,6 +122,13 @@ func Run(cmd *cobra.Command, args []string) {
 		}()
 	}
 
+	// To help users notice issues with the agent, we show the error messages in
+	// the agent pod's events.
+	eventf, err := newEventf(config.InstallNS)
+	if err != nil {
+		logs.Log.Fatalf("failed to create event recorder: %v", err)
+	}
+
 	dataGatherers := map[string]datagatherer.DataGatherer{}
 	group, gctx := errgroup.WithContext(ctx)
 
@@ -180,7 +194,7 @@ func Run(cmd *cobra.Command, args []string) {
 	// configured output using data in datagatherer caches or refreshing from
 	// APIs each cycle depending on datagatherer implementation
 	for {
-		gatherAndOutputData(config, preflightClient, dataGatherers)
+		gatherAndOutputData(eventf, config, preflightClient, dataGatherers)
 
 		if config.OneShot {
 			break
@@ -190,7 +204,44 @@ func Run(cmd *cobra.Command, args []string) {
 	}
 }
 
-func gatherAndOutputData(config CombinedConfig, preflightClient client.Client, dataGatherers map[string]datagatherer.DataGatherer) {
+// Creates an event recorder for the agent's Pod object. Expects the env var
+// POD_NAME to contain the pod name. Note that the RBAC rule allowing sending
+// events is attached to the pod's service account, not the impersonated service
+// account (venafi-connection).
+func newEventf(installNS string) (Eventf, error) {
+	restcfg, err := kubeconfig.LoadRESTConfig("")
+	if err != nil {
+		logs.Log.Fatalf("failed to load kubeconfig: %v", err)
+	}
+	scheme := runtime.NewScheme()
+	_ = corev1.AddToScheme(scheme)
+
+	var eventf Eventf
+	if os.Getenv("POD_NAME") == "" {
+		eventf = func(eventType, reason, msg string, args ...interface{}) {}
+		logs.Log.Printf("error messages will not show in the pod's events because the POD_NAME environment variable is empty")
+	} else {
+		podName := os.Getenv("POD_NAME")
+
+		eventClient, err := kubernetes.NewForConfig(restcfg)
+		if err != nil {
+			return eventf, fmt.Errorf("failed to create event client: %v", err)
+		}
+		broadcaster := record.NewBroadcaster()
+		broadcaster.StartRecordingToSink(&clientgocorev1.EventSinkImpl{Interface: eventClient.CoreV1().Events(installNS)})
+		eventRec := broadcaster.NewRecorder(scheme, corev1.EventSource{})
+		eventf = func(eventType, reason, msg string, args ...interface{}) {
+			eventRec.Eventf(&corev1.Pod{ObjectMeta: v1.ObjectMeta{Name: podName, Namespace: installNS}}, eventType, reason, msg, args...)
+		}
+	}
+
+	return eventf, nil
+}
+
+// Like Printf but for sending events to the agent's Pod object.
+type Eventf func(eventType, reason, msg string, args ...interface{})
+
+func gatherAndOutputData(eventf Eventf, config CombinedConfig, preflightClient client.Client, dataGatherers map[string]datagatherer.DataGatherer) {
 	var readings []*api.DataReading
 
 	if config.InputPath != "" {
@@ -226,6 +277,7 @@ func gatherAndOutputData(config CombinedConfig, preflightClient client.Client, d
 			return postData(config, preflightClient, readings)
 		}
 		err := backoff.RetryNotify(post, backOff, func(err error, t time.Duration) {
+			eventf("Warning", "PushingErr", "retrying in %v after error: %s", t, err)
 			logs.Log.Printf("retrying in %v after error: %s", t, err)
 		})
 		if err != nil {