loading of external shared-object plugins

circle.yml

@@ -4,9 +4,9 @@ machine:
   post:
     - sudo service zookeeper stop
     - go version
-    - go version | grep 1.7.5 || sudo rm -rf /usr/local/go
-    - wget https://storage.googleapis.com/golang/go1.7.5.linux-amd64.tar.gz
-    - sudo tar -C /usr/local -xzf go1.7.5.linux-amd64.tar.gz
+    - sudo rm -rf /usr/local/go
+    - wget https://storage.googleapis.com/golang/go1.8rc3.linux-amd64.tar.gz
+    - sudo tar -C /usr/local -xzf go1.8rc3.linux-amd64.tar.gz
     - go version
 
 dependencies:

cmd/telegraf/telegraf.go

@@ -6,19 +6,25 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"path"
+	"path/filepath"
+	"plugin"
 	"runtime"
 	"strings"
 	"syscall"
 
 	"github.com/influxdata/telegraf/agent"
 	"github.com/influxdata/telegraf/internal/config"
 	"github.com/influxdata/telegraf/logger"
+	"github.com/influxdata/telegraf/registry"
+	"github.com/influxdata/telegraf/registry/inputs"
+	"github.com/influxdata/telegraf/registry/outputs"
+
 	_ "github.com/influxdata/telegraf/plugins/aggregators/all"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	_ "github.com/influxdata/telegraf/plugins/inputs/all"
-	"github.com/influxdata/telegraf/plugins/outputs"
 	_ "github.com/influxdata/telegraf/plugins/outputs/all"
 	_ "github.com/influxdata/telegraf/plugins/processors/all"
+
 	"github.com/kardianos/service"
 )
 
@@ -50,6 +56,8 @@ var fUsage = flag.String("usage", "",
 	"print usage for a plugin, ie, 'telegraf -usage mysql'")
 var fService = flag.String("service", "",
 	"operate on the service")
+var fPlugins = flag.String("plugins", "",
+	"path to directory containing external plugins")
 
 // Telegraf version, populated linker.
 //   ie, -ldflags "-X main.version=`git describe --always --tags`"
@@ -246,11 +254,62 @@ func (p *program) Stop(s service.Service) error {
 	return nil
 }
 
+// loadExternalPlugins loads external plugins from shared libraries (.so, .dll, etc.)
+// in the specified directory.
+func loadExternalPlugins(rootDir string) error {
+	return filepath.Walk(rootDir, func(pth string, info os.FileInfo, err error) error {
+		// Stop if there was an error.
+		if err != nil {
+			return err
+		}
+
+		// Ignore directories.
+		if info.IsDir() {
+			return nil
+		}
+
+		// Ignore files that aren't shared libraries.
+		ext := strings.ToLower(path.Ext(pth))
+		if ext != ".so" && ext != ".dll" {
+			return nil
+		}
+
+		// name will be the path to the plugin file beginning at the root
+		// directory, minus the extension.
+		// ie, if the plugin file is /opt/telegraf-plugins/group1/foo.so, name
+		//     will be "group1/foo"
+		name := strings.TrimPrefix(strings.TrimPrefix(pth, rootDir), string(os.PathSeparator))
+		name = strings.TrimSuffix(name, filepath.Ext(pth))
+		registry.SetName("external" + string(os.PathSeparator) + name)
+		defer registry.SetName("")
+
+		// Load plugin.
+		_, err = plugin.Open(pth)
+		if err != nil {
+			return fmt.Errorf("error loading [%s]: %s", pth, err)
+		}
+
+		return nil
+	})
+}
+
 func main() {
 	flag.Usage = func() { usageExit(0) }
 	flag.Parse()
 	args := flag.Args()
 
+	// Load external plugins, if requested.
+	if *fPlugins != "" {
+		pluginsDir, err := filepath.Abs(*fPlugins)
+		if err != nil {
+			log.Fatal("E! " + err.Error())
+		}
+		log.Printf("I! Loading external plugins from: %s\n", pluginsDir)
+		if err := loadExternalPlugins(*fPlugins); err != nil {
+			log.Fatal("E! " + err.Error())
+		}
+	}
+
 	inputFilters, outputFilters := []string{}, []string{}
 	if *fInputFilters != "" {
 		inputFilters = strings.Split(":"+strings.TrimSpace(*fInputFilters)+":", ":")

internal/config/config.go

@@ -18,12 +18,12 @@ import (
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/models"
-	"github.com/influxdata/telegraf/plugins/aggregators"
-	"github.com/influxdata/telegraf/plugins/inputs"
-	"github.com/influxdata/telegraf/plugins/outputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
-	"github.com/influxdata/telegraf/plugins/processors"
 	"github.com/influxdata/telegraf/plugins/serializers"
+	"github.com/influxdata/telegraf/registry/aggregators"
+	"github.com/influxdata/telegraf/registry/inputs"
+	"github.com/influxdata/telegraf/registry/outputs"
+	"github.com/influxdata/telegraf/registry/processors"
 
 	"github.com/influxdata/config"
 	"github.com/influxdata/toml"
@@ -40,6 +40,14 @@ var (
 
 	// envVarRe is a regex to find environment variables in the config file
 	envVarRe = regexp.MustCompile(`\$\w+`)
+
+	// addQuoteRe is a regex for finding and adding quotes around / characters
+	// when they are used for distinguishing external plugins.
+	// ie, a ReplaceAll() with this pattern will be used to turn this:
+	//     [[inputs.external/test/example]]
+	//  to
+	//     [[inputs."external/test/example"]]
+	addQuoteRe = regexp.MustCompile(`(\[?\[?inputs|outputs|processors|aggregators)\.(external\/[^.\]]+)`)
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -701,6 +709,9 @@ func parseFile(fpath string) (*ast.Table, error) {
 		}
 	}
 
+	// add quotes around external plugin paths.
+	contents = addQuoteRe.ReplaceAll(contents, []byte(`$1."$2"`))
+
 	return toml.Parse(contents)
 }
 

internal/config/config_test.go

@@ -6,11 +6,11 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf/internal/models"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/inputs/exec"
 	"github.com/influxdata/telegraf/plugins/inputs/memcached"
 	"github.com/influxdata/telegraf/plugins/inputs/procstat"
 	"github.com/influxdata/telegraf/plugins/parsers"
+	"github.com/influxdata/telegraf/registry/inputs"
 
 	"github.com/stretchr/testify/assert"
 )

plugins/aggregators/minmax/minmax.go

@@ -2,7 +2,7 @@ package minmax
 
 import (
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/aggregators"
+	"github.com/influxdata/telegraf/registry/aggregators"
 )
 
 type MinMax struct {

plugins/inputs/aerospike/aerospike.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal/errchan"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 
 	as "github.com/aerospike/aerospike-client-go"
 )

plugins/inputs/apache/apache.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type Apache struct {

plugins/inputs/bcache/bcache.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type Bcache struct {

plugins/inputs/cassandra/cassandra.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 	"io/ioutil"
 	"log"
 	"net/http"

plugins/inputs/ceph/ceph.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 const (

plugins/inputs/cgroup/cgroup.go

@@ -2,7 +2,7 @@ package cgroup
 
 import (
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type CGroup struct {

plugins/inputs/chrony/chrony.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 var (

plugins/inputs/cloudwatch/cloudwatch.go

@@ -15,7 +15,7 @@ import (
 	internalaws "github.com/influxdata/telegraf/internal/config/aws"
 	"github.com/influxdata/telegraf/internal/errchan"
 	"github.com/influxdata/telegraf/internal/limiter"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type (

plugins/inputs/conntrack/conntrack.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 	"log"
 	"path/filepath"
 )

plugins/inputs/consul/consul.go

@@ -6,7 +6,7 @@ import (
 	"github.com/hashicorp/consul/api"
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type Consul struct {

plugins/inputs/couchbase/couchbase.go

@@ -3,7 +3,7 @@ package couchbase
 import (
 	couchbase "github.com/couchbase/go-couchbase"
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 	"sync"
 )
 

plugins/inputs/couchdb/couchdb.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 	"net/http"
 	"reflect"
 	"strings"

plugins/inputs/disque/disque.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type Disque struct {

plugins/inputs/dns_query/dns_query.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal/errchan"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type DnsQuery struct {

plugins/inputs/docker/docker.go

@@ -17,7 +17,7 @@ import (
 	"github.com/docker/engine-api/types"
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 // Docker object

plugins/inputs/dovecot/dovecot.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal/errchan"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type Dovecot struct {

plugins/inputs/elasticsearch/elasticsearch.go

@@ -11,8 +11,8 @@ import (
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/errchan"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	jsonparser "github.com/influxdata/telegraf/plugins/parsers/json"
+	"github.com/influxdata/telegraf/registry/inputs"
 	"io/ioutil"
 	"strings"
 )

plugins/inputs/exec/exec.go

@@ -16,9 +16,9 @@ import (
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/errchan"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	"github.com/influxdata/telegraf/plugins/parsers"
 	"github.com/influxdata/telegraf/plugins/parsers/nagios"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 const sampleConfig = `

plugins/inputs/filestat/filestat.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal/globpath"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 const sampleConfig = `

plugins/inputs/graylog/graylog.go

@@ -16,7 +16,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 type ResponseMetrics struct {

plugins/inputs/haproxy/haproxy.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 //CSV format: https://cbonte.github.io/haproxy-dconv/1.5/configuration.html#9.1

plugins/inputs/hddtemp/hddtemp.go

@@ -4,8 +4,8 @@ package hddtemp
 
 import (
 	"github.com/influxdata/telegraf"
-	"github.com/influxdata/telegraf/plugins/inputs"
 	gohddtemp "github.com/influxdata/telegraf/plugins/inputs/hddtemp/go-hddtemp"
+	"github.com/influxdata/telegraf/registry/inputs"
 )
 
 const defaultAddress = "127.0.0.1:7634"