Generate emails and github/gitlab issue bodies from a template

CHANGES.md

@@ -1,3 +1,12 @@
+UNRELEASED
+==========
+
+Features
+--------
+
+- Allow configuration of the body of created Github/Gitlab issues via a template in the configuration file. ([\#84](https://github.com/matrix-org/rageshake/issues/84))
+
+
 1.11.0 (2023-08-11)
 ===================
 

README.md

@@ -18,6 +18,11 @@ Optional parameters:
  * `-listen <address>`: TCP network address to listen for HTTP requests
    on. Example: `:9110`.
 
+## Issue template
+
+It is possible to override the templates used to construct emails, and Github and Gitlab issues.
+See [templates/README.md](templates/README.md) for more information.
+
 ## HTTP endpoints
 
 The following HTTP endpoints are exposed:

main.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/google/go-github/github"
@@ -36,6 +37,17 @@ import (
 
 	"gopkg.in/yaml.v2"
 )
+import _ "embed"
+
+// DefaultIssueBodyTemplate is the default template used for `issue_body_template_file` in the config.
+//
+//go:embed templates/issue_body.tmpl
+var DefaultIssueBodyTemplate string
+
+// DefaultEmailBodyTemplate is the default template used for `email_body_template_file` in the config.
+//
+//go:embed templates/email_body.tmpl
+var DefaultEmailBodyTemplate string
 
 var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.")
 var bindAddr = flag.String("listen", ":9110", "The port to listen on.")
@@ -63,6 +75,9 @@ type config struct {
 	GitlabProjectLabels     map[string][]string `yaml:"gitlab_project_labels"`
 	GitlabIssueConfidential bool                `yaml:"gitlab_issue_confidential"`
 
+	IssueBodyTemplateFile string `yaml:"issue_body_template_file"`
+	EmailBodyTemplateFile string `yaml:"email_body_template_file"`
+
 	SlackWebhookURL string `yaml:"slack_webhook_url"`
 
 	EmailAddresses []string `yaml:"email_addresses"`
@@ -158,7 +173,17 @@ func main() {
 	log.Printf("Using %s/listing as public URI", apiPrefix)
 
 	rand.Seed(time.Now().UnixNano())
-	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, appNameMap, cfg})
+	http.Handle("/api/submit", &submitServer{
+		issueTemplate:        parseTemplate(DefaultIssueBodyTemplate, cfg.IssueBodyTemplateFile, "issue"),
+		emailTemplate:        parseTemplate(DefaultEmailBodyTemplate, cfg.EmailBodyTemplateFile, "email"),
+		ghClient:             ghClient,
+		glClient:             glClient,
+		apiPrefix:            apiPrefix,
+		slack:                slack,
+		genericWebhookClient: genericWebhookClient,
+		allowedAppNameMap:    appNameMap,
+		cfg:                  cfg,
+	})
 
 	// Make sure bugs directory exists
 	_ = os.Mkdir("bugs", os.ModePerm)
@@ -186,6 +211,28 @@ func main() {
 	log.Fatal(http.ListenAndServe(*bindAddr, nil))
 }
 
+// parseTemplate parses a template file, with fallback to default.
+//
+// If `templateFilePath` is non-empty, it is used as the name of a file to read. Otherwise, `defaultTemplate` is
+// used.
+//
+// The template text is then parsed into a template named `templateName`.
+func parseTemplate(defaultTemplate string, templateFilePath string, templateName string) *template.Template {
+	templateText := defaultTemplate
+	if templateFilePath != "" {
+		issueTemplateBytes, err := os.ReadFile(templateFilePath)
+		if err != nil {
+			log.Fatalf("Unable to read template file `%s`: %s", templateFilePath, err)
+		}
+		templateText = string(issueTemplateBytes)
+	}
+	parsedTemplate, err := template.New(templateName).Parse(templateText)
+	if err != nil {
+		log.Fatalf("Invalid template file %s in config file: %s", templateFilePath, err)
+	}
+	return parsedTemplate
+}
+
 func configureAppNameMap(cfg *config) map[string]bool {
 	if len(cfg.AllowedAppNames) == 0 {
 		fmt.Println("Warning: allowed_app_names is empty. Accepting requests from all app names")

rageshake.sample.yaml

@@ -55,8 +55,12 @@ smtp_server: localhost:25
 smtp_username: myemailuser
 smtp_password: myemailpass
 
-
 # a list of webhook URLs, (see docs/generic_webhook.md)
 generic_webhook_urls: 
   - https://server.example.com/your-server/api
   - http://another-server.com/api
+
+# The paths of template files for the body of Github and Gitlab issues, and emails.
+# See `templates/README.md` for more information.
+issue_body_template_file: path/to/issue_body.tmpl
+email_body_template_file: path/to/email_body.tmpl

submit.go

@@ -37,6 +37,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/google/go-github/github"
@@ -47,6 +48,12 @@ import (
 var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
 
 type submitServer struct {
+	// Template for building github and gitlab issues
+	issueTemplate *template.Template
+
+	// Template for building emails
+	emailTemplate *template.Template
+
 	// github client for reporting bugs. may be nil, in which case,
 	// reporting is disabled.
 	ghClient *github.Client
@@ -78,6 +85,16 @@ type jsonLogEntry struct {
 	Lines string `json:"lines"`
 }
 
+// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template` and
+// `email_body_template`.
+//
+// !!! Keep in step with the documentation in `templates/README.md` !!!
+type issueBodyTemplatePayload struct {
+	payload
+	// Complete link to the listing URL that contains all uploaded logs
+	ListingURL string
+}
+
 // Stores additional information created during processing of a payload
 type genericWebhookPayload struct {
 	payload
@@ -87,7 +104,10 @@ type genericWebhookPayload struct {
 	ListingURL string `json:"listing_url"`
 }
 
-// Stores information about a request made to this server
+// `payload` stores information about a request made to this server.
+//
+// !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step
+// with the documentation in `templates/README.md` !!!
 type payload struct {
 	// A unique ID for this payload, generated within this server 
 	ID         string            `json:"id"`
@@ -505,7 +525,7 @@ func (s *submitServer) saveReport(ctx context.Context, p payload, reportDir, lis
 		return nil, err
 	}
 
-	if err := s.sendEmail(p, reportDir); err != nil {
+	if err := s.sendEmail(p, reportDir, listingURL); err != nil {
 		return nil, err
 	}
 
@@ -580,9 +600,12 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing
 	}
 	owner, repo := splits[0], splits[1]
 
-	issueReq := buildGithubIssueRequest(p, listingURL)
+	issueReq, err := buildGithubIssueRequest(p, listingURL, s.issueTemplate)
+	if err != nil {
+		return err
+	}
 
-	issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, &issueReq)
+	issue, _, err := s.ghClient.Issues.Create(ctx, owner, repo, issueReq)
 	if err != nil {
 		return err
 	}
@@ -602,7 +625,10 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub
 	glProj := s.cfg.GitlabProjectMappings[p.AppName]
 	glLabels := s.cfg.GitlabProjectLabels[p.AppName]
 
-	issueReq := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential)
+	issueReq, err := buildGitlabIssueRequest(p, listingURL, s.issueTemplate, glLabels, s.cfg.GitlabIssueConfidential)
+	if err != nil {
+		return err
+	}
 
 	issue, _, err := s.glClient.Issues.CreateIssue(glProj, issueReq)
 
@@ -649,80 +675,72 @@ func buildReportTitle(p payload) string {
 	return trimmedUserText
 }
 
-func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
+func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title string, body []byte, err error) {
 	var bodyBuf bytes.Buffer
-	fmt.Fprintf(&bodyBuf, "User message:\n\n%s\n\n", p.UserText)
-	var dataKeys []string
-	for k := range p.Data {
-		dataKeys = append(dataKeys, k)
-	}
-	sort.Strings(dataKeys)
-	for _, k := range dataKeys {
-		v := p.Data[k]
-		fmt.Fprintf(&bodyBuf, "%s: %s%s%s%s", k, quoteChar, v, quoteChar, newline)
-	}
-
-	return &bodyBuf
-}
 
-func buildGenericIssueRequest(p payload, listingURL string) (title, body string) {
-	bodyBuf := buildReportBody(p, "  \n", "`")
-
-	// Add log links to the body
-	fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL)
-	fmt.Fprintf(bodyBuf, " ([archive](%s))", listingURL+"?format=tar.gz")
+	issuePayload := issueBodyTemplatePayload{
+		payload:    p,
+		ListingURL: listingURL,
+	}
 
-	for _, file := range p.Files {
-		fmt.Fprintf(
-			bodyBuf,
-			" / [%s](%s)",
-			file,
-			listingURL+"/"+file,
-		)
+	if err = bodyTemplate.Execute(&bodyBuf, issuePayload); err != nil {
+		return
 	}
 
 	title = buildReportTitle(p)
-
-	body = bodyBuf.String()
+	body = bodyBuf.Bytes()
 
 	return
 }
 
-func buildGithubIssueRequest(p payload, listingURL string) github.IssueRequest {
-	title, body := buildGenericIssueRequest(p, listingURL)
+func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (*github.IssueRequest, error) {
+	title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate)
+	if err != nil {
+		return nil, err
+	}
 
 	labels := p.Labels
 	// go-github doesn't like nils
 	if labels == nil {
 		labels = []string{}
 	}
-	return github.IssueRequest{
+	bodyStr := string(body)
+	return &github.IssueRequest{
 		Title:  &title,
-		Body:   &body,
+		Body:   &bodyStr,
 		Labels: &labels,
-	}
+	}, nil
 }
 
-func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) *gitlab.CreateIssueOptions {
-	title, body := buildGenericIssueRequest(p, listingURL)
+func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *template.Template, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) {
+	title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate)
+	if err != nil {
+		return nil, err
+	}
 
 	if p.Labels != nil {
 		labels = append(labels, p.Labels...)
 	}
 
+	bodyStr := string(body)
 	return &gitlab.CreateIssueOptions{
 		Title:        &title,
-		Description:  &body,
+		Description:  &bodyStr,
 		Confidential: &confidential,
 		Labels:       labels,
-	}
+	}, nil
 }
 
-func (s *submitServer) sendEmail(p payload, reportDir string) error {
+func (s *submitServer) sendEmail(p payload, reportDir string, listingURL string) error {
 	if len(s.cfg.EmailAddresses) == 0 {
 		return nil
 	}
 
+	title, body, err := buildGenericIssueRequest(p, listingURL, s.emailTemplate)
+	if err != nil {
+		return err
+	}
+
 	e := email.NewEmail()
 
 	e.From = "Rageshake <[email protected]>"
@@ -731,10 +749,8 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error {
 	}
 
 	e.To = s.cfg.EmailAddresses
-
-	e.Subject = fmt.Sprintf("[%s] %s", p.AppName, buildReportTitle(p))
-
-	e.Text = buildReportBody(p, "\n", "\"").Bytes()
+	e.Subject = fmt.Sprintf("[%s] %s", p.AppName, title)
+	e.Text = body
 
 	allFiles := append(p.Files, p.Logs...)
 	for _, file := range allFiles {
@@ -746,7 +762,7 @@ func (s *submitServer) sendEmail(p payload, reportDir string) error {
 	if s.cfg.SMTPPassword != "" || s.cfg.SMTPUsername != "" {
 		auth = smtp.PlainAuth("", s.cfg.SMTPUsername, s.cfg.SMTPPassword, s.cfg.SMTPServer)
 	}
-	err := e.Send(s.cfg.SMTPServer, auth)
+	err = e.Send(s.cfg.SMTPServer, auth)
 	if err != nil {
 		return err
 	}