include_dir support

config.go

@@ -12,6 +12,8 @@ import (
 	"context"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/HeavyHorst/remco/backends"
@@ -29,33 +31,72 @@ type resource struct {
 
 // configuration is the representation of an config file
 type configuration struct {
-	LogLevel  string `toml:"log_level"`
-	LogFormat string `toml:"log_format"`
-	Resource  []resource
+	LogLevel   string `toml:"log_level"`
+	LogFormat  string `toml:"log_format"`
+	IncludeDir string `toml:"include_dir"`
+	Resource   []resource
+}
+
+func readFileAndExpandEnv(path string) ([]byte, error) {
+	buf, err := ioutil.ReadFile(path)
+	if err != nil {
+		return buf, err
+	}
+	// expand the environment variables
+	buf = []byte(os.ExpandEnv(string(buf)))
+	return buf, nil
 }
 
 // newConfiguration reads the file at `path`, expand the environment variables
 // and unmarshals it to a new configuration struct.
 // it returns an error if any.
 func newConfiguration(path string) (configuration, error) {
 	var c configuration
-	buf, err := ioutil.ReadFile(path)
+
+	buf, err := readFileAndExpandEnv(path)
 	if err != nil {
 		return c, err
 	}
-	buf = []byte(os.ExpandEnv(string(buf)))
+
 	if err := toml.Unmarshal(buf, &c); err != nil {
 		return c, err
 	}
 
-	c.loadGlobals()
+	c.configureLogger()
+
+	if c.IncludeDir != "" {
+		files, err := ioutil.ReadDir(c.IncludeDir)
+		if err != nil {
+			return c, err
+		}
+		for _, file := range files {
+			if strings.HasSuffix(file.Name(), ".toml") {
+				fp := filepath.Join(c.IncludeDir, file.Name())
+				log.WithFields(logrus.Fields{
+					"path": fp,
+				}).Info("Loading resource configuration")
+				buf, err := readFileAndExpandEnv(fp)
+				if err != nil {
+					return c, err
+				}
+				var r resource
+				if err := toml.Unmarshal(buf, &r); err != nil {
+					return c, err
+				}
+				// don't add empty resources
+				if len(r.Template) > 0 {
+					c.Resource = append(c.Resource, r)
+				}
+			}
+		}
+	}
 
 	return c, nil
 }
 
-// loadGlobals configures remco with the global configuration options
+// configureLogger configures the global logger
 // for example it sets the log level and log formatting
-func (c *configuration) loadGlobals() {
+func (c *configuration) configureLogger() {
 	if c.LogLevel != "" {
 		err := log.SetLevel(c.LogLevel)
 		if err != nil {

configWatcher.go

@@ -9,102 +9,140 @@
 package main
 
 import (
-	"context"
-	"fmt"
+	"strings"
+	"sync"
 	"time"
 
-	"github.com/HeavyHorst/easyKV"
 	"github.com/HeavyHorst/remco/log"
+	"github.com/fsnotify/fsnotify"
 )
 
 // the configWatcher watches the config file for changes
 type configWatcher struct {
-	stoppedW  chan struct{}
-	stopWatch chan struct{}
-	filePath  string
-	cancel    context.CancelFunc
+	stoppedW         chan struct{}
+	stopWatch        chan struct{}
+	stopWatchConf    chan struct{}
+	stoppedWatchConf chan struct{}
+	reloadChan       chan struct{}
+	wg               sync.WaitGroup
+	mu               sync.Mutex
+	canceled         bool
+	configPath       string
 }
 
 // call c.run in its own goroutine
 // and write to the w.stoppedW chan if its done
 func (w *configWatcher) runConfig(c configuration) {
-	go func() {
-		defer func() {
-			w.stoppedW <- struct{}{}
-		}()
-		c.run(w.stopWatch)
+	defer func() {
+		w.stoppedW <- struct{}{}
 	}()
+	c.run(w.stopWatch)
 }
 
-// reload stops the old config and starts a new one
-// this function blocks forever if runConfig was never called before
-func (w *configWatcher) reload() {
-	defer func() {
-		// we may try to send on the closed channel w.stopWatch
-		// we need to recover from this panic
-		if r := recover(); r != nil {
-			if fmt.Sprintf("%v", r) != "send on closed channel" {
-				panic(r)
-			}
-		}
-	}()
+func (w *configWatcher) getCanceled() bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.canceled
+}
+
+// startWatchConfig starts to watch the configuration file and all files under include_dir which ends with .toml
+// if there is an event (write, remove, create) we write to w.reloadChan to trigger an relaod and return.
+func (w *configWatcher) startWatchConfig(config configuration) {
+	w.wg.Add(1)
+	defer w.wg.Done()
 
-	newConf, err := newConfiguration(w.filePath)
+	watcher, err := fsnotify.NewWatcher()
 	if err != nil {
 		log.Error(err)
-		return
 	}
-	// stop the old watcher and wait until it has stopped
-	w.stopWatch <- struct{}{}
-	<-w.stoppedW
-	// start a new watcher
-	w.runConfig(newConf)
-}
+	defer watcher.Close()
 
-func newConfigWatcher(filepath string, watcher easyKV.ReadWatcher, config configuration, done chan struct{}) *configWatcher {
-	w := &configWatcher{
-		stoppedW:  make(chan struct{}),
-		stopWatch: make(chan struct{}),
-		filePath:  filepath,
+	// add the configfile to the watcher
+	err = watcher.Add(w.configPath)
+	if err != nil {
+		log.Error(err)
 	}
 
-	w.runConfig(config)
+	// add the include_dir to the watcher
+	if config.IncludeDir != "" {
+		err = watcher.Add(config.IncludeDir)
+		if err != nil {
+			log.Error(err)
+		}
+	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	w.cancel = cancel
+	// watch the config for changes
+	for {
+		select {
+		case <-w.stopWatchConf:
+			return
+		case event := <-watcher.Events:
+			if event.Op&fsnotify.Write == fsnotify.Write ||
+				event.Op&fsnotify.Remove == fsnotify.Remove ||
+				event.Op&fsnotify.Create == fsnotify.Create {
+				// only watch .toml files
+				if strings.HasSuffix(event.Name, ".toml") || event.Name == w.configPath {
+					time.Sleep(500 * time.Millisecond)
 
-	reload := make(chan struct{})
-	go func() {
-		// watch the config for changes
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			default:
-				_, err := watcher.WatchPrefix("", ctx, easyKV.WithKeys([]string{""}))
-				if err != nil {
-					if err != easyKV.ErrWatchCanceled {
-						log.Error(err)
-						time.Sleep(2 * time.Second)
+					// don't try to reload if w is already canceled
+					if w.getCanceled() {
+						return
 					}
-					continue
+
+					w.reloadChan <- struct{}{}
+					return
 				}
-				time.Sleep(1 * time.Second)
-				reload <- struct{}{}
 			}
 		}
-	}()
+	}
+}
 
+func newConfigWatcher(configPath string, config configuration, done chan struct{}) *configWatcher {
+	w := &configWatcher{
+		stoppedW:      make(chan struct{}),
+		stopWatch:     make(chan struct{}),
+		stopWatchConf: make(chan struct{}),
+		reloadChan:    make(chan struct{}),
+		configPath:    configPath,
+	}
+
+	go w.runConfig(config)
+	go w.startWatchConfig(config)
+	w.wg.Add(1)
 	go func() {
+		defer w.wg.Done()
 		for {
 			select {
-			case <-reload:
-				w.reload()
+			case <-w.reloadChan:
+				newConf, err := newConfiguration(w.configPath)
+				if err != nil {
+					log.Error(err)
+					continue
+				}
+
+				// don't try to relaod anything if w is already canceled
+				if w.getCanceled() {
+					continue
+				}
+
+				// stop the old config and wait until it has stopped
+				w.stopWatch <- struct{}{}
+				<-w.stoppedW
+				go w.runConfig(newConf)
+
+				// restart the fsnotify config watcher (the include_dir folder may have changed)
+				// startWatchConfig returns when calling reload (we don't need to stop it)
+				go w.startWatchConfig(newConf)
 			case <-w.stoppedW:
+				// close the reloadChan
+				// every attempt to write to reloadChan would block forever otherwise
+				close(w.reloadChan)
+
+				// close the done channel
+				// this signals the main function that the configWatcher has completed all work
+				// for example all backends are configured with onetime=true
 				close(done)
-				// there is no runnign runConfig function which can answer to the stop method
-				// we need to send to the w.stoppedW channel so that we don't block
-				w.stoppedW <- struct{}{}
+				return
 			}
 		}
 	}()
@@ -113,7 +151,12 @@ func newConfigWatcher(filepath string, watcher easyKV.ReadWatcher, config config
 }
 
 func (w *configWatcher) stop() {
-	w.cancel()
+	w.mu.Lock()
+	w.canceled = true
+	w.mu.Unlock()
 	close(w.stopWatch)
-	<-w.stoppedW
+	close(w.stopWatchConf)
+
+	// wait for the main routine and startWatchConfig to exit
+	w.wg.Wait()
 }

docs/config.toml

@@ -52,15 +52,20 @@ canonifyurls = true
 	url    = "config/sample"
 	weight = 20
 
+[[menu.main]]
+	name   = "Sample resource configuration"
+	url    = "config/sampleresource"
+    weight = 30
+
 [[menu.main]]
 	name   = "Template"
 	url    = "template/"
-	weight = 30
+	weight = 40
 
 [[menu.main]]
 	name   = "License"
 	url    = "license/"
-	weight = 40
+	weight = 50
 
 
 [blackfriday]

docs/content/config/index.md

@@ -16,6 +16,8 @@ to configure values, you can simply use $VARIABLE_NAME or ${VARIABLE_NAME} and t
    - Valid levels are panic, fatal, error, warn, info and debug. Default is info.
  - **log_format(string):** 
    - The format of the log messages. Valid formats are *text* and *json*.
+ - **include_dir(string):**
+   - Specify an entire directory of resource configuration files to include.
 
 ## Template configuration options
  - **src(string):**