add plugin system

Makefile

@@ -18,9 +18,13 @@ lint:
 	pre-commit run --all-files
 
 .PHONY: test
-test:
+test: build-testfilter
 	go test --tags=test -coverprofile cover.out -count=1 -race -p 4 -v ./...
 
+.PHONY: build-testfilter
+build-testfilter:
+	env CGO_ENABLED=0 go build --tags=testfilter $(LDFLAGS) -o /tmp/ct-monitor/testfilter ./filter/t/main.go
+
 .PHONY: setup-container-structure-test
 setup-container-structure-test:
 	if [ -z "$(shell which container-structure-test)" ]; then \

README.md

@@ -14,6 +14,44 @@ Flags:
   -h, --help              help for ct-monitor
 ```
 
+## Plugins
+Custom plugins can be specified to filter issuances or perform any extra work with the issuances detected. For instance, you may want to get certificate issuances for `example.com` including wildcard and subdomains, but ignore issuances for the `dev.example.com` subdomain only. Better yet, you can use plugins to implement your own mailer or send notifications to Slack instead of using the built-in mailer.
+
+A plugin simply needs to implement the `IssuanceFilter` interface via `net/rpc`.
+
+For instance, this plugin simply prints out the number of issuances and otherwise does not modify the slice of Issuance objects.
+
+```go
+package main
+
+import (
+	"github.com/Hsn723/certspotter-client/api"
+	"github.com/Hsn723/ct-monitor/filter"
+	"github.com/cybozu-go/log"
+	"github.com/hashicorp/go-plugin"
+)
+
+type sampleFilter struct{}
+
+func (sampleFilter) Filter(is []api.Issuance) ([]api.Issuance, error) {
+	_ = log.Info("running sample filter", map[string]interface{}{
+		"issuances": len(is),
+	})
+	return is, nil
+}
+
+func main() {
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: filter.HandshakeConfig,
+		Plugins: map[string]plugin.Plugin{
+			filter.PluginKey: &filter.IssuanceFilterPlugin{Impl: &sampleFilter{}},
+		},
+	})
+}
+```
+
+For more detailed examples, refer to the documentation of [HashiCorp's go-plugin](https://github.com/hashicorp/go-plugin).
+
 ## Example config
 ```toml
 [alert_config]

cmd/root.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/Hsn723/certspotter-client/api"
 	"github.com/Hsn723/ct-monitor/config"
+	"github.com/Hsn723/ct-monitor/filter"
 	"github.com/Hsn723/ct-monitor/mailer"
 	"github.com/cybozu-go/log"
 	"github.com/spf13/cobra"
@@ -81,16 +82,16 @@ func sendMail(mailSender mailer.Mailer, domain string, issuances []api.Issuance)
 	return mailSender.Send(subject, body)
 }
 
-func checkIssuances(domain string, wildcards, subdomains bool, c api.CertspotterClient, mailSender mailer.Mailer) error {
-	key := getDomainConfigName(domain)
+func checkIssuances(dc config.DomainConfig, c api.CertspotterClient, mailSender mailer.Mailer, fc config.FilterConfig) error {
+	key := getDomainConfigName(dc.Name)
 	lastIssuance := position.GetUint64(key)
-	issuances, err := c.GetIssuances(domain, wildcards, subdomains, lastIssuance)
+	issuances, err := c.GetIssuances(dc.Name, dc.MatchWildcards, dc.IncludeSubdomains, lastIssuance)
 	if err != nil {
 		return err
 	}
 	if len(issuances) == 0 {
 		_ = log.Info("no new issuances observed", map[string]interface{}{
-			"domain": domain,
+			"domain": dc.Name,
 		})
 		return nil
 	}
@@ -102,12 +103,23 @@ func checkIssuances(domain string, wildcards, subdomains bool, c api.Certspotter
 			"sha256": issuance.Cert.SHA256,
 		})
 	}
-	if err := sendMail(mailSender, domain, issuances); err != nil {
+	issuances, err = filter.ApplyFilters(fc.Filters, issuances)
+	if err != nil {
+		_ = log.Info("errors encountered running filters", map[string]interface{}{
+			"error":   err.Error(),
+			"domain":  dc.Name,
+			"filters": fc.Filters,
+		})
+	}
+	if len(issuances) == 0 {
+		return nil
+	}
+	if err := sendMail(mailSender, dc.Name, issuances); err != nil {
 		return err
 	}
 	position.Set(key, lastIssuance)
 	_ = log.Info("done checking", map[string]interface{}{
-		"domain": domain,
+		"domain": dc.Name,
 	})
 	return nil
 }
@@ -130,6 +142,21 @@ func atomicWritePosition(pc config.PositionConfig) error {
 	return os.Rename(tmpFile.Name(), pc.Filename)
 }
 
+func getMailSenderForDomain(conf *config.Config, dc config.DomainConfig, defaultMailSender mailer.Mailer) mailer.Mailer {
+	if dc.Mailer == "" {
+		return defaultMailSender
+	}
+	domainMailer := conf.GetMailer(dc.Mailer)
+	if err := domainMailer.Init(); err != nil {
+		_ = log.Error("could not initialize domain mailer, using default", map[string]interface{}{
+			"error":  err.Error(),
+			"domain": dc.Name,
+		})
+		return defaultMailSender
+	}
+	return domainMailer
+}
+
 func runRoot(cmd *cobra.Command, args []string) error {
 	_ = log.Info("ct-monitor", map[string]interface{}{
 		"version":  version,
@@ -154,11 +181,8 @@ func runRoot(cmd *cobra.Command, args []string) error {
 		Token:    conf.Token,
 	}
 	for _, domain := range conf.Domains {
-		domainMailer := defaultMailSender
-		if domain.Mailer != "" {
-			domainMailer = conf.GetMailer(domain.Mailer)
-		}
-		if err := checkIssuances(domain.Name, domain.MatchWildcards, domain.IncludeSubdomains, csp, domainMailer); err != nil {
+		domainMailer := getMailSenderForDomain(conf, domain, defaultMailSender)
+		if err := checkIssuances(domain, csp, domainMailer, conf.FilterConfig); err != nil {
 			_ = log.Error(err.Error(), map[string]interface{}{
 				"domain": domain.Name,
 			})

config/config.go

@@ -36,6 +36,8 @@ type Config struct {
 	Sendgrid mailer.SendgridMailer `mapstructure:"sendgrid"`
 	// SMTP represents the mailer configuration for using plain SMTP.
 	SMTP mailer.SMTPMailer `mapstructure:"smtp"`
+	// FilterConfig represent filter plugin configuration.
+	FilterConfig FilterConfig `mapstructure:"filter_config"`
 }
 
 // DomainConfig contains domain configurations.
@@ -64,6 +66,10 @@ type PositionConfig struct {
 	Filename string `mapstructure:"filename"`
 }
 
+type FilterConfig struct {
+	Filters []string `mapstructure:"filters"`
+}
+
 // Mailer represents a mailer name.
 type Mailer string
 

config/config_test.go

@@ -59,6 +59,7 @@ func TestLoad(t *testing.T) {
 					To:     "[email protected]",
 					APIKey: "hoge",
 				},
+				FilterConfig: FilterConfig{Filters: []string{}},
 			},
 		},
 		{

config/t/full.toml

@@ -13,8 +13,11 @@ certspotter_token = "dummy"
 [alert_config]
     mailer_config = "sendgrid"
 
-[position_config]
-    filename = "positions.toml"
+    [position_config]
+        filename = "positions.toml"
+
+[filter_config]
+    filters = []
 
 [smtp]
     from = "[email protected]"

filter/common.go

@@ -0,0 +1,59 @@
+package filter
+
+import (
+	"github.com/Hsn723/certspotter-client/api"
+	"github.com/hashicorp/go-plugin"
+	"net/rpc"
+)
+
+var (
+	HandshakeConfig = plugin.HandshakeConfig{
+		ProtocolVersion:  1,
+		MagicCookieKey:   "CT_MONITOR_PLUGIN",
+		MagicCookieValue: "issuance_filter",
+	}
+	PluginKey = "issuance"
+	PluginMap = map[string]plugin.Plugin{
+		PluginKey: &IssuanceFilterPlugin{},
+	}
+)
+
+// IssuanceFilter is the interface exposed as a plugin.
+type IssuanceFilter interface {
+	Filter(is []api.Issuance) ([]api.Issuance, error)
+}
+
+// IssuanceFilterRPC is a plugin implementation over RPC.
+type IssuanceFilterRPCClient struct {
+	client *rpc.Client
+}
+
+func (f *IssuanceFilterRPCClient) Filter(is []api.Issuance) ([]api.Issuance, error) {
+	var resp []api.Issuance
+	err := f.client.Call("Plugin.Filter", is, &resp)
+	return resp, err
+}
+
+// IssuanceFilterRPCServer is the RPC server that IssuanceFilterRPC talks to.
+type IssuanceFilterRPCServer struct {
+	Impl IssuanceFilter
+}
+
+func (s *IssuanceFilterRPCServer) Filter(is []api.Issuance, resp *[]api.Issuance) error {
+	r, err := s.Impl.Filter(is)
+	*resp = r
+	return err
+}
+
+// IssuanceFilterPlugin us an implementation of plugin.Plugin.
+type IssuanceFilterPlugin struct {
+	Impl IssuanceFilter
+}
+
+func (p *IssuanceFilterPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
+	return &IssuanceFilterRPCServer{Impl: p.Impl}, nil
+}
+
+func (*IssuanceFilterPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
+	return &IssuanceFilterRPCClient{client: c}, nil
+}

filter/filter.go

@@ -0,0 +1,48 @@
+package filter
+
+import (
+	"os/exec"
+	"time"
+
+	"github.com/Hsn723/certspotter-client/api"
+	"github.com/hashicorp/go-plugin"
+)
+
+// ApplyFilters runs the filter plugins and returns the resulting issuances.
+// Plugin errors are ignored. It is up to the plugin to log them appropriately.
+func ApplyFilters(filterPaths []string, issuances []api.Issuance) ([]api.Issuance, error) {
+	res := issuances
+	for _, fp := range filterPaths {
+		r, err := applyFilter(fp, res)
+		if err != nil {
+			return res, err
+		}
+		res = r
+	}
+	return res, nil
+}
+
+func applyFilter(filter string, issuances []api.Issuance) ([]api.Issuance, error) {
+	client := plugin.NewClient(&plugin.ClientConfig{
+		HandshakeConfig:  HandshakeConfig,
+		Plugins:          PluginMap,
+		Cmd:              exec.Command(filter),
+		AllowedProtocols: []plugin.Protocol{plugin.ProtocolNetRPC},
+		StartTimeout:     10 * time.Second,
+		Managed:          true,
+	})
+	defer client.Kill()
+
+	rpcClient, err := client.Client()
+	if err != nil {
+		return issuances, err
+	}
+
+	raw, err := rpcClient.Dispense(PluginKey)
+	if err != nil {
+		return issuances, err
+	}
+
+	issuanceFilter := raw.(IssuanceFilter)
+	return issuanceFilter.Filter(issuances)
+}