Add file watch to support config reload on file change

CHANGELOG.md

@@ -15,6 +15,7 @@
 
 - Add semconv 1.7.0 and 1.8.0 (#4452)
 - Added `feature-gates` CLI flag for controlling feature gate state. (#4368)
+- Add file watch to support config reload on file change (#4454) 
 
 ## v0.39.0 Beta
 

config/configmapprovider/file.go

@@ -18,12 +18,17 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
+	"time"
 
 	"go.opentelemetry.io/collector/config"
 )
 
+var errNoOp = errors.New("no change")
+
 type fileMapProvider struct {
 	fileName string
+	watching bool
 }
 
 // NewFile returns a new Provider that reads the configuration from the given file.
@@ -33,7 +38,7 @@ func NewFile(fileName string) Provider {
 	}
 }
 
-func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (Retrieved, error) {
+func (fmp *fileMapProvider) Retrieve(ctx context.Context, onChange func(*ChangeEvent)) (Retrieved, error) {
 	if fmp.fileName == "" {
 		return nil, errors.New("config file not specified")
 	}
@@ -43,9 +48,65 @@ func (fmp *fileMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (R
 		return nil, fmt.Errorf("error loading config file %q: %w", fmp.fileName, err)
 	}
 
+	// Ensure that watches are only created once as Retrieve is called each time that `onChange()` is invoked.
+	if !fmp.watching {
+		watchFile(ctx, fmp.fileName, onChange)
+		fmp.watching = true
+	}
+
 	return &simpleRetrieved{confMap: cp}, nil
 }
 
-func (*fileMapProvider) Shutdown(context.Context) error {
+func (*fileMapProvider) Shutdown(_ context.Context) error {
 	return nil
 }
+
+// watchFile sets up a watch on a filename to detect changes and calls onChange() with a suitable ChangeEvent.
+// The watch is cancelled when the context is done.
+func watchFile(ctx context.Context, filename string, onChange func(*ChangeEvent)) {
+	var (
+		lastfi os.FileInfo
+	)
+
+	check := func() error {
+		currfi, err := os.Stat(filename)
+		modified := false
+		if err != nil {
+			if os.IsNotExist(err) && lastfi != nil {
+				return errNoOp
+			}
+			return err
+		}
+		if lastfi != nil && (currfi.Size() != lastfi.Size() || !currfi.ModTime().Equal(lastfi.ModTime())) {
+			modified = true
+		}
+
+		lastfi = currfi
+		if modified {
+			return nil
+		}
+
+		return errNoOp
+	}
+
+	// Check the file every second and if it's been updated then initiate a config reload.
+	go func() {
+		ticker := time.NewTicker(time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				// If check returns a valid event, exit the loop. A new watch will be placed on the next Retrieve()
+				err := check()
+				if err == nil || err != errNoOp {
+					time.Sleep(time.Second * 2)
+					onChange(&ChangeEvent{
+						Error: err,
+					})
+				}
+			}
+		}
+	}()
+}

config/configmapprovider/file_test.go

@@ -0,0 +1,81 @@
+// Copyright  The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package configmapprovider
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWatchFile(t *testing.T) {
+	// Create a temporary file.
+	file, err := ioutil.TempFile("", "file_watcher_test")
+	require.NoError(t, err)
+	defer func() {
+		file.Close()
+		os.Remove(file.Name())
+	}()
+
+	received := atomic.Value{}
+	received.Store(false)
+
+	// Write some initial content.
+	_, err = file.WriteString("hello")
+	require.NoError(t, err)
+	require.NoError(t, file.Sync())
+
+	// Setup the watch.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	watchFile(ctx, file.Name(), testingOnChange(t, &received))
+	time.Sleep(time.Second * 2)
+
+	// Update the file and verify we see the updated content.
+	_, err = file.WriteString(" world")
+	require.NoError(t, err)
+	require.NoError(t, file.Sync())
+
+	require.Eventually(t, func() bool {
+		return received.Load().(bool)
+	}, time.Second*10, time.Second)
+
+	// Cancel the context.
+	cancel()
+}
+
+func TestWatchFile_ReloadError(t *testing.T) {
+	// Create then delete a temporary file so we have a filename that we know can't be opened.
+	file, err := ioutil.TempFile("", "file_watcher_test")
+	require.NoError(t, err)
+	_ = file.Close()
+	require.NoError(t, os.Remove(file.Name()))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	watchFile(ctx, file.Name(), func(event *ChangeEvent) {})
+}
+
+func testingOnChange(t *testing.T, r *atomic.Value) func(c *ChangeEvent) {
+	return func(c *ChangeEvent) {
+		require.Nil(t, c.Error)
+		r.Store(true)
+	}
+}

config/configmapprovider/properties.go

@@ -40,7 +40,7 @@ func NewProperties(properties []string) Provider {
 	}
 }
 
-func (pmp *propertiesMapProvider) Retrieve(_ context.Context, onChange func(*ChangeEvent)) (Retrieved, error) {
+func (pmp *propertiesMapProvider) Retrieve(_ context.Context, _ func(*ChangeEvent)) (Retrieved, error) {
 	if len(pmp.properties) == 0 {
 		return &simpleRetrieved{confMap: config.NewMap()}, nil
 	}

service/collector.go

@@ -35,13 +35,13 @@ import (
 	"go.uber.org/zap"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
 	"go.opentelemetry.io/collector/config/configunmarshaler"
 	"go.opentelemetry.io/collector/extension/ballastextension"
 	"go.opentelemetry.io/collector/service/internal"
 	"go.opentelemetry.io/collector/service/internal/telemetrylogs"
 )
 
-// State defines Collector's state.
 type State int
 
 const (
@@ -106,7 +106,6 @@ func New(set CollectorSettings) (*Collector, error) {
 		set:   set,
 		state: state,
 	}, nil
-
 }
 
 // GetState returns current state of the collector server.
@@ -147,16 +146,24 @@ LOOP:
 		select {
 		case err := <-col.cfgW.watcher:
 			col.logger.Warn("Config updated", zap.Error(err))
+			if err != nil {
+				goto LOOP
+			}
 
-			col.setCollectorState(Closing)
-
-			if err = col.cfgW.close(ctx); err != nil {
-				return fmt.Errorf("failed to close config watcher: %w", err)
+			// Get the config and ensure that it is valid. If it is not, keep running in the last valid state and
+			// keep trying until the configuration is fixed.
+			cfg, err := col.cfgW.get()
+			if err != nil {
+				col.logger.Error("unable to parse configuration", zap.Error(err))
+				goto LOOP
 			}
+
+			col.setCollectorState(Closing)
 			if err = col.service.Shutdown(ctx); err != nil {
 				return fmt.Errorf("failed to shutdown the retiring config: %w", err)
 			}
-			if err = col.setupConfigurationComponents(ctx); err != nil {
+
+			if err = col.setupConfigurationComponents(ctx, cfg); err != nil {
 				return fmt.Errorf("failed to setup configuration components: %w", err)
 			}
 		case err := <-col.asyncErrorChannel:
@@ -175,24 +182,20 @@ LOOP:
 
 // setupConfigurationComponents loads the config and starts the components. If all the steps succeeds it
 // sets the col.service with the service currently running.
-func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
-	col.setCollectorState(Starting)
-
+func (col *Collector) setupConfigurationComponents(ctx context.Context, cfg *config.Config) error {
 	var err error
-	if col.cfgW, err = newConfigWatcher(ctx, col.set); err != nil {
-		return err
-	}
 
-	if col.logger, err = telemetrylogs.NewLogger(col.cfgW.cfg.Service.Telemetry.Logs, col.set.LoggingOptions); err != nil {
+	if col.logger, err = telemetrylogs.NewLogger(cfg.Service.Telemetry.Logs, col.set.LoggingOptions); err != nil {
 		return fmt.Errorf("failed to get logger: %w", err)
 	}
 
 	col.logger.Info("Applying configuration...")
+	col.setCollectorState(Starting)
 
 	col.service, err = newService(&svcSettings{
 		BuildInfo: col.set.BuildInfo,
 		Factories: col.set.Factories,
-		Config:    col.cfgW.cfg,
+		Config:    cfg,
 		Telemetry: component.TelemetrySettings{
 			Logger:         col.logger,
 			TracerProvider: col.tracerProvider,
@@ -210,6 +213,7 @@ func (col *Collector) setupConfigurationComponents(ctx context.Context) error {
 	}
 
 	return nil
+
 }
 
 // Run starts the collector according to the given configuration given, and waits for it to complete.
@@ -228,7 +232,14 @@ func (col *Collector) Run(ctx context.Context) error {
 
 	col.asyncErrorChannel = make(chan error)
 
-	if err := col.setupConfigurationComponents(ctx); err != nil {
+	// set up config watcher
+	col.cfgW = newConfigWatcher(ctx, col.set)
+	cfg, err := col.cfgW.get()
+	if err != nil {
+		return fmt.Errorf("unable to get config: %w", err)
+	}
+
+	if err := col.setupConfigurationComponents(ctx, cfg); err != nil {
 		return err
 	}
 
@@ -254,9 +265,8 @@ func (col *Collector) shutdown(ctx context.Context) error {
 	// Begin shutdown sequence.
 	col.logger.Info("Starting shutdown...")
 
-	if err := col.cfgW.close(ctx); err != nil {
-		errs = multierr.Append(errs, fmt.Errorf("failed to close config provider watcher: %w", err))
-	}
+	// Close the config watcher
+	col.cfgW.close()
 
 	if err := col.set.ConfigMapProvider.Shutdown(ctx); err != nil {
 		errs = multierr.Append(errs, fmt.Errorf("failed to shutdown config provider: %w", err))

service/config_watcher.go

@@ -24,36 +24,54 @@ import (
 )
 
 type configWatcher struct {
-	cfg     *config.Config
-	ret     configmapprovider.Retrieved
+	ctx     context.Context
+	set     CollectorSettings
 	watcher chan error
+	ret     configmapprovider.Retrieved
 }
 
-func newConfigWatcher(ctx context.Context, set CollectorSettings) (*configWatcher, error) {
-	cm := &configWatcher{watcher: make(chan error, 1)}
+func newConfigWatcher(ctx context.Context, set CollectorSettings) *configWatcher {
+	cm := &configWatcher{
+		ctx:     ctx,
+		watcher: make(chan error, 1),
+		set:     set,
+	}
+
+	return cm
+}
 
-	ret, err := set.ConfigMapProvider.Retrieve(ctx, cm.onChange)
+func (cm *configWatcher) get() (*config.Config, error) {
+	// Ensure that a previously existing Retrieved is closed out properly.
+	if cm.ret != nil {
+		err := cm.ret.Close(cm.ctx)
+		if err != nil {
+			return nil, fmt.Errorf("cannot close previously retrieved config: %w", err)
+		}
+	}
+
+	var (
+		cfg *config.Config
+		err error
+	)
+	cm.ret, err = cm.set.ConfigMapProvider.Retrieve(cm.ctx, cm.onChange)
 	if err != nil {
 		return nil, fmt.Errorf("cannot retrieve the configuration: %w", err)
 	}
 
-	var cfg *config.Config
-	m, err := ret.Get(ctx)
+	m, err := cm.ret.Get(cm.ctx)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get the configuration: %w", err)
 	}
-	if cfg, err = set.ConfigUnmarshaler.Unmarshal(m, set.Factories); err != nil {
+
+	if cfg, err = cm.set.ConfigUnmarshaler.Unmarshal(m, cm.set.Factories); err != nil {
 		return nil, fmt.Errorf("cannot unmarshal the configuration: %w", err)
 	}
 
 	if err = cfg.Validate(); err != nil {
 		return nil, fmt.Errorf("invalid configuration: %w", err)
 	}
 
-	cm.cfg = cfg
-	cm.ret = ret
-
-	return cm, nil
+	return cfg, nil
 }
 
 func (cm *configWatcher) onChange(event *configmapprovider.ChangeEvent) {
@@ -62,7 +80,6 @@ func (cm *configWatcher) onChange(event *configmapprovider.ChangeEvent) {
 	}
 }
 
-func (cm *configWatcher) close(ctx context.Context) error {
+func (cm *configWatcher) close() {
 	close(cm.watcher)
-	return cm.ret.Close(ctx)
 }