Add error TTL setting

x-pack/libbeat/common/cloudfoundry/cache.go

@@ -22,21 +22,23 @@ type cfClient interface {
 
 // clientCacheWrap wraps the cloudfoundry client to add a cache in front of GetAppByGuid.
 type clientCacheWrap struct {
-	cache  *common.Cache
-	client cfClient
-	log    *logp.Logger
+	cache    *common.Cache
+	client   cfClient
+	log      *logp.Logger
+	errorTTL time.Duration
 }
 
 // newClientCacheWrap creates a new cache for application data.
-func newClientCacheWrap(client cfClient, ttl time.Duration, log *logp.Logger) *clientCacheWrap {
+func newClientCacheWrap(client cfClient, ttl time.Duration, errorTTL time.Duration, log *logp.Logger) *clientCacheWrap {
 	return &clientCacheWrap{
-		cache:  common.NewCacheWithExpireOnAdd(ttl, 100),
-		client: client,
-		log:    log,
+		cache:    common.NewCacheWithExpireOnAdd(ttl, 100),
+		client:   client,
+		errorTTL: errorTTL,
+		log:      log,
 	}
 }
 
-type cachedAppResponse struct {
+type appResponse struct {
 	app *cfclient.App
 	err error
 }
@@ -45,15 +47,17 @@ type cachedAppResponse struct {
 // stores it in the internal cache
 func (c *clientCacheWrap) fetchAppByGuid(guid string) (*cfclient.App, error) {
 	app, err := c.client.GetAppByGuid(guid)
-	resp := cachedAppResponse{
+	resp := appResponse{
 		app: &app,
 		err: err,
 	}
+	timeout := time.Duration(0)
 	if err != nil {
 		// Cache nil, because is what we want to return when there was an error
 		resp.app = nil
+		timeout = c.errorTTL
 	}
-	c.cache.Put(guid, &resp)
+	c.cache.PutWithTimeout(guid, &resp, timeout)
 	return resp.app, resp.err
 }
 
@@ -64,7 +68,7 @@ func (c *clientCacheWrap) GetAppByGuid(guid string) (*cfclient.App, error) {
 	if cachedResp == nil {
 		return c.fetchAppByGuid(guid)
 	}
-	resp, ok := cachedResp.(*cachedAppResponse)
+	resp, ok := cachedResp.(*appResponse)
 	if !ok {
 		return nil, fmt.Errorf("error converting cached app response (of type %T), this is likely a bug", cachedResp)
 	}

x-pack/libbeat/common/cloudfoundry/cache_test.go

@@ -25,7 +25,7 @@ func TestClientCacheWrap(t *testing.T) {
 		Memory: 1, // use this field to track if from cache or from client
 	}
 	fakeClient := &fakeCFClient{app, 0}
-	cache := newClientCacheWrap(fakeClient, ttl, logp.NewLogger("cloudfoundry"))
+	cache := newClientCacheWrap(fakeClient, ttl, ttl, logp.NewLogger("cloudfoundry"))
 
 	missingAppGuid := mustCreateFakeGuid()
 

x-pack/libbeat/common/cloudfoundry/config.go

@@ -41,8 +41,11 @@ type Config struct {
 	// multiple filebeats will shard the load of receiving and sending events.
 	ShardID string `config:"shard_id"`
 
-	// Maximum amount of time to cache application objects from CF client
+	// Maximum amount of time to cache application objects from CF client.
 	CacheDuration time.Duration `config:"cache_duration"`
+
+	// Time to wait before retrying to get application info in case of error.
+	CacheRetryDelay time.Duration `config:"cache_retry_delay"`
 }
 
 // InitDefaults initialize the defaults for the configuration.
@@ -55,6 +58,7 @@ func (c *Config) InitDefaults() {
 	}
 	c.ShardID = uuid.String()
 	c.CacheDuration = 120 * time.Second
+	c.CacheRetryDelay = 20 * time.Second
 	c.Version = ConsumerVersionV1
 }
 

x-pack/libbeat/common/cloudfoundry/hub.go

@@ -67,7 +67,7 @@ func (h *Hub) Client() (Client, error) {
 	if h.cfg.UaaAddress != "" {
 		cf.Endpoint.AuthEndpoint = h.cfg.UaaAddress
 	}
-	return newClientCacheWrap(cf, h.cfg.CacheDuration, h.log), nil
+	return newClientCacheWrap(cf, h.cfg.CacheDuration, h.cfg.CacheRetryDelay, h.log), nil
 }
 
 // RlpListener returns a listener client that calls the passed callback when the provided events are streamed through

x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc

@@ -53,6 +53,8 @@ It has the following settings:
 
 `client_secret`:: Client Secret to authenticate with Cloud Foundry.
 
-`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata.
+`cache_duration`:: (Optional) Maximum amount of time to cache an application's metadata. Defaults to 120 seconds.
+
+`cache_retry_delay`:: (Optional) Time to wait before trying to obtain an application's metadata again in case of error. Defaults to 20 seconds.
 
 `ssl`:: (Optional) SSL configuration to use when connecting to Cloud Foundry.