feat(config): Add framework for migrating deprecated plugins (#13377)

cmd/telegraf/cmd_config.go

@@ -0,0 +1,174 @@
+// Command handling for configuration "config" command
+package main
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/logger"
+	"github.com/influxdata/telegraf/migrations"
+	"github.com/urfave/cli/v2"
+)
+
+func getConfigCommands(pluginFilterFlags []cli.Flag, outputBuffer io.Writer) []*cli.Command {
+	return []*cli.Command{
+		{
+			Name:  "config",
+			Usage: "commands for generating and migrating configurations",
+			Flags: pluginFilterFlags,
+			Action: func(cCtx *cli.Context) error {
+				// The sub_Filters are populated when the filter flags are set after the subcommand config
+				// e.g. telegraf config --section-filter inputs
+				filters := processFilterFlags(cCtx)
+
+				printSampleConfig(outputBuffer, filters)
+				return nil
+			},
+			Subcommands: []*cli.Command{
+				{
+					Name:  "create",
+					Usage: "create a full sample configuration and show it",
+					Description: `
+The 'create' produces a full configuration containing all plugins as an example
+and shows it on the console. You may apply 'section' or 'plugin' filtering
+to reduce the output to the plugins you need
+
+Create the full configuration
+
+> telegraf config create
+
+To produce a configuration only containing a Modbus input plugin and an
+InfluxDB v2 output plugin use
+
+> telegraf config create --section-filter "inputs:outputs" --input-filter "modbus" --output-filter "influxdb_v2"
+`,
+					Flags: pluginFilterFlags,
+					Action: func(cCtx *cli.Context) error {
+						filters := processFilterFlags(cCtx)
+
+						printSampleConfig(outputBuffer, filters)
+						return nil
+					},
+				},
+				{
+					Name:  "migrate",
+					Usage: "migrate deprecated plugins and options of the configuration(s)",
+					Description: `
+The 'migrate' command reads the configuration files specified via '--config' or
+'--config-directory' and tries to migrate plugins or options that are currently
+deprecated using the recommended replacements. If no configuration file is
+explicitly specified the command reads the default locations and uses those
+configuration files. Migrated files are stored with a '.migrated' suffix at the
+location of the  inputs. If you are migrating remote configurations the migrated
+configurations is stored in the current directory using the filename of the URL
+with a '.migrated' suffix.
+It is highly recommended to test those migrated configurations before using
+those files unattended!
+
+To migrate the file 'mysettings.conf' use
+
+> telegraf --config mysettings.conf config migrate
+`,
+					Flags: []cli.Flag{
+						&cli.BoolFlag{
+							Name:  "force",
+							Usage: "forces overwriting of an existing migration file",
+						},
+					},
+					Action: func(cCtx *cli.Context) error {
+						// Setup logging
+						telegraf.Debug = cCtx.Bool("debug")
+						logConfig := logger.LogConfig{Debug: telegraf.Debug}
+						if err := logger.SetupLogging(logConfig); err != nil {
+							return err
+						}
+
+						// Check if we have migrations at all. There might be
+						// none if you run a custom build without migrations
+						// enabled.
+						if len(migrations.PluginMigrations) == 0 {
+							return errors.New("no migrations available")
+						}
+						log.Printf("%d plugin migration(s) available", len(migrations.PluginMigrations))
+
+						// Collect the given configuration files
+						configFiles := cCtx.StringSlice("config")
+						configDir := cCtx.StringSlice("config-directory")
+						for _, fConfigDirectory := range configDir {
+							files, err := config.WalkDirectory(fConfigDirectory)
+							if err != nil {
+								return err
+							}
+							configFiles = append(configFiles, files...)
+						}
+
+						// If no "config" or "config-directory" flag(s) was
+						// provided we should load default configuration files
+						if len(configFiles) == 0 {
+							paths, err := config.GetDefaultConfigPath()
+							if err != nil {
+								return err
+							}
+							configFiles = paths
+						}
+
+						for _, fn := range configFiles {
+							log.Printf("D! Trying to migrate %q...", fn)
+
+							// Read and parse the config file
+							data, remote, err := config.LoadConfigFile(fn)
+							if err != nil {
+								return fmt.Errorf("opening input %q failed: %w", fn, err)
+							}
+
+							out, applied, err := config.ApplyMigrations(data)
+							if err != nil {
+								return err
+							}
+
+							// Do not write a migration file if nothing was done
+							if applied == 0 {
+								log.Printf("I! No migration applied for %q", fn)
+								continue
+							}
+
+							// Construct the output filename
+							// For remote locations we just save the filename
+							// with the migrated suffix.
+							outfn := fn + ".migrated"
+							if remote {
+								u, err := url.Parse(fn)
+								if err != nil {
+									return fmt.Errorf("parsing remote config URL %q failed: %w", fn, err)
+								}
+								outfn = filepath.Base(u.Path) + ".migrated"
+							}
+
+							log.Printf("I! %d migration applied for %q, writing result as %q", applied, fn, outfn)
+
+							// Make sure the file does not exist yet if we should not overwrite
+							if !cCtx.Bool("force") {
+								if _, err := os.Stat(outfn); !errors.Is(err, os.ErrNotExist) {
+									return fmt.Errorf("output file %q already exists", outfn)
+								}
+							}
+
+							// Write the output file
+							if err := os.WriteFile(outfn, out, 0640); err != nil {
+								return fmt.Errorf("writing output %q failed: %w", outfn, err)
+							}
+						}
+						return nil
+					},
+				},
+			},
+		},
+	}
+}

cmd/telegraf/cmd_secretstore.go

@@ -1,4 +1,4 @@
-// Command handling for secret-stores' "secret" command
+// Command handling for secret-stores' "secrets" command
 package main
 
 import (

cmd/telegraf/main.go

@@ -245,6 +245,11 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 		return m.Run()
 	}
 
+	commands := append(
+		getConfigCommands(pluginFilterFlags, outputBuffer),
+		getSecretStoreCommands(m)...,
+	)
+
 	app := &cli.App{
 		Name:   "Telegraf",
 		Usage:  "The plugin-driven server agent for collecting & reporting metrics.",
@@ -341,19 +346,6 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 			}, extraFlags...),
 		Action: action,
 		Commands: append([]*cli.Command{
-			{
-				Name:  "config",
-				Usage: "print out full sample configuration to stdout",
-				Flags: pluginFilterFlags,
-				Action: func(cCtx *cli.Context) error {
-					// The sub_Filters are populated when the filter flags are set after the subcommand config
-					// e.g. telegraf config --section-filter inputs
-					filters := processFilterFlags(cCtx)
-
-					printSampleConfig(outputBuffer, filters)
-					return nil
-				},
-			},
 			{
 				Name:  "version",
 				Usage: "print current version to stdout",
@@ -362,9 +354,7 @@ func runApp(args []string, outputBuffer io.Writer, pprof Server, c TelegrafConfi
 					return nil
 				},
 			},
-		},
-			getSecretStoreCommands(m)...,
-		),
+		}, commands...),
 	}
 
 	// Make sure we safely erase secrets

config/config.go

@@ -376,7 +376,7 @@ func WalkDirectory(path string) ([]string, error) {
 //  1. $TELEGRAF_CONFIG_PATH
 //  2. $HOME/.telegraf/telegraf.conf
 //  3. /etc/telegraf/telegraf.conf and /etc/telegraf/telegraf.d/*.conf
-func getDefaultConfigPath() ([]string, error) {
+func GetDefaultConfigPath() ([]string, error) {
 	envfile := os.Getenv("TELEGRAF_CONFIG_PATH")
 	homefile := os.ExpandEnv("${HOME}/.telegraf/telegraf.conf")
 	etcfile := "/etc/telegraf/telegraf.conf"
@@ -434,7 +434,7 @@ func (c *Config) LoadConfig(path string) error {
 	paths := []string{}
 
 	if path == "" {
-		if paths, err = getDefaultConfigPath(); err != nil {
+		if paths, err = GetDefaultConfigPath(); err != nil {
 			return err
 		}
 	} else {
@@ -446,7 +446,7 @@ func (c *Config) LoadConfig(path string) error {
 			log.Printf("I! Loading config: %s", path)
 		}
 
-		data, err := LoadConfigFile(path)
+		data, _, err := LoadConfigFile(path)
 		if err != nil {
 			return fmt.Errorf("error loading config file %s: %w", path, err)
 		}
@@ -696,33 +696,37 @@ func trimBOM(f []byte) []byte {
 	return bytes.TrimPrefix(f, []byte("\xef\xbb\xbf"))
 }
 
-func LoadConfigFile(config string) ([]byte, error) {
+// LoadConfigFile loads the content of a configuration file and returns it
+// together with a flag denoting if the file is from a remote location such
+// as a web server.
+func LoadConfigFile(config string) ([]byte, bool, error) {
 	if fetchURLRe.MatchString(config) {
 		u, err := url.Parse(config)
 		if err != nil {
-			return nil, err
+			return nil, true, err
 		}
 
 		switch u.Scheme {
 		case "https", "http":
-			return fetchConfig(u)
+			data, err := fetchConfig(u)
+			return data, true, err
 		default:
-			return nil, fmt.Errorf("scheme %q not supported", u.Scheme)
+			return nil, true, fmt.Errorf("scheme %q not supported", u.Scheme)
 		}
 	}
 
 	// If it isn't a https scheme, try it as a file
 	buffer, err := os.ReadFile(config)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	mimeType := http.DetectContentType(buffer)
 	if !strings.Contains(mimeType, "text/plain") {
-		return nil, fmt.Errorf("provided config is not a TOML file: %s", config)
+		return nil, false, fmt.Errorf("provided config is not a TOML file: %s", config)
 	}
 
-	return buffer, nil
+	return buffer, false, nil
 }
 
 func fetchConfig(u *url.URL) ([]byte, error) {

config/config_test.go

@@ -3,6 +3,8 @@ package config_test
 import (
 	"bytes"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -441,6 +443,21 @@ func TestConfig_AzureMonitorNamespacePrefix(t *testing.T) {
 	}
 }
 
+func TestGetDefaultConfigPathFromEnvURL(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer ts.Close()
+
+	c := config.NewConfig()
+	t.Setenv("TELEGRAF_CONFIG_PATH", ts.URL)
+	configPath, err := config.GetDefaultConfigPath()
+	require.NoError(t, err)
+	require.Equal(t, []string{ts.URL}, configPath)
+	err = c.LoadConfig("")
+	require.NoError(t, err)
+}
+
 func TestConfig_URLLikeFileName(t *testing.T) {
 	c := config.NewConfig()
 	err := c.LoadConfig("http:##www.example.com.conf")

config/internal_test.go

@@ -135,18 +135,3 @@ func TestURLRetries3FailsThenPasses(t *testing.T) {
 	require.NoError(t, c.LoadConfig(ts.URL))
 	require.Equal(t, 4, responseCounter)
 }
-
-func TestConfig_getDefaultConfigPathFromEnvURL(t *testing.T) {
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer ts.Close()
-
-	c := NewConfig()
-	t.Setenv("TELEGRAF_CONFIG_PATH", ts.URL)
-	configPath, err := getDefaultConfigPath()
-	require.NoError(t, err)
-	require.Equal(t, []string{ts.URL}, configPath)
-	err = c.LoadConfig("")
-	require.NoError(t, err)
-}