update tproxy logging messages to be a bit more informative (#514)

* update logging in endpoints controller so that it is a bit more descriptive to help with debugging

CHANGELOG.md

@@ -34,6 +34,9 @@ IMPROVEMENTS:
 * Connect: support upgrades for services deployed before endpoints controller to
   upgrade to a version of consul-k8s with endpoints controller. [[GH-509](https://github.com/hashicorp/consul-k8s/pull/509)]
 
+* Connect: add additional logging to the endpoints controller and connect-init command to help
+  the user debug if pods arent starting right away. [[GH-514](https://github.com/hashicorp/consul-k8s/pull/514/)]
+
 BUG FIXES:
 * Connect: Use `runAsNonRoot: false` for connect-init's container when tproxy is enabled. [[GH-493](https://github.com/hashicorp/consul-k8s/pull/493)]
 * CRDs: Fix a bug where the `config` field in `ProxyDefaults` CR was not synced to Consul because

connect-inject/endpoints_controller.go

@@ -140,22 +140,22 @@ func (r *EndpointsController) Reconcile(ctx context.Context, req ctrl.Request) (
 					r.Log.Error(err, "failed to get pod", "name", address.TargetRef.Name)
 					return ctrl.Result{}, err
 				}
+				podHostIP := pod.Status.HostIP
 
 				if hasBeenInjected(pod) {
 					// Build the endpointAddressMap up for deregistering service instances later.
 					endpointAddressMap[pod.Status.PodIP] = true
 					// Create client for Consul agent local to the pod.
-					client, err := r.remoteConsulClient(pod.Status.HostIP, r.consulNamespace(pod.Namespace))
+					client, err := r.remoteConsulClient(podHostIP, r.consulNamespace(pod.Namespace))
 					if err != nil {
-						r.Log.Error(err, "failed to create a new Consul client", "address", pod.Status.HostIP)
+						r.Log.Error(err, "failed to create a new Consul client", "address", podHostIP)
 						return ctrl.Result{}, err
 					}
 
 					var managedByEndpointsController bool
 					if raw, ok := pod.Labels[keyManagedBy]; ok && raw == managedByValue {
 						managedByEndpointsController = true
 					}
-
 					// For pods managed by this controller, create and register the service instance.
 					if managedByEndpointsController {
 						// Get information from the pod to create service instance registrations.
@@ -169,7 +169,8 @@ func (r *EndpointsController) Reconcile(ctx context.Context, req ctrl.Request) (
 						// Note: the order of how we register services is important,
 						// and the connect-proxy service should come after the "main" service
 						// because its alias health check depends on the main service existing.
-						r.Log.Info("registering service with Consul", "name", serviceRegistration.Name)
+						r.Log.Info("registering service with Consul", "name", serviceRegistration.Name,
+							"id", serviceRegistration.ID, "agentIP", podHostIP)
 						err = client.Agent().ServiceRegister(serviceRegistration)
 						if err != nil {
 							r.Log.Error(err, "failed to register service", "name", serviceRegistration.Name)

subcommand/connect-init/command.go

@@ -152,8 +152,10 @@ func (c *Command) Run(args []string) int {
 	// Now wait for the service to be registered. Do this by querying the Agent for a service
 	// which maps to this pod+namespace.
 	var proxyID string
+	registrationRetryCount := 0
 	var errServiceNameMismatch error
 	err = backoff.Retry(func() error {
+		registrationRetryCount++
 		filter := fmt.Sprintf("Meta[%q] == %q and Meta[%q] == %q", connectinject.MetaKeyPodName, c.flagPodName, connectinject.MetaKeyKubeNS, c.flagPodNamespace)
 		serviceList, err := consulClient.Agent().ServicesWithFilter(filter)
 		if err != nil {
@@ -163,6 +165,11 @@ func (c *Command) Run(args []string) int {
 		// Wait for the service and the connect-proxy service to be registered.
 		if len(serviceList) != 2 {
 			c.logger.Info("Unable to find registered services; retrying")
+			// Once every 10 times we're going to print this informational message to the pod logs so that
+			// it is not "lost" to the user at the end of the retries when the pod enters a CrashLoop.
+			if registrationRetryCount%10 == 0 {
+				c.logger.Info("Check to ensure a Kubernetes service has been created for this application.")
+			}
 			return fmt.Errorf("did not find correct number of services: %d", len(serviceList))
 		}
 		for _, svc := range serviceList {