Import rule URLs, add them to markdown & JSON output (chainguard-dev#165

)

* Add URLs to markdown output

* run prettier against output

pkg/bincapz/bincapz.go

@@ -9,8 +9,15 @@ type Behavior struct {
 	MatchStrings []string `json:",omitempty" yaml:",omitempty"`
 	RiskScore    int
 	RiskLevel    string `json:",omitempty" yaml:",omitempty"`
-	RuleAuthor   string `json:",omitempty" yaml:",omitempty"`
-	RuleLicense  string `json:",omitempty" yaml:",omitempty"`
+
+	RuleURL      string `json:",omitempty" yaml:",omitempty"`
+	ReferenceURL string `json:",omitempty" yaml:",omitempty"`
+
+	RuleAuthor    string `json:",omitempty" yaml:",omitempty"`
+	RuleAuthorURL string `json:",omitempty" yaml:",omitempty"`
+
+	RuleLicense    string `json:",omitempty" yaml:",omitempty"`
+	RuleLicenseURL string `json:",omitempty" yaml:",omitempty"`
 
 	DiffAdded   bool `json:",omitempty" yaml:",omitempty"`
 	DiffRemoved bool `json:",omitempty" yaml:",omitempty"`

pkg/render/markdown.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"sort"
 	"strings"
 
@@ -26,6 +27,20 @@ func mdRisk(score int, level string) string {
 	return fmt.Sprintf("%s %s", riskEmoji(score), level)
 }
 
+// generate a markdown link for a matched fragment.
+func matchFragmentLink(s string) string {
+	// it's probably the name of a matched YARA field, for example, if it's xor'ed data
+	if strings.HasPrefix(s, "$") {
+		return s
+	}
+
+	if strings.HasPrefix(s, "https:") || strings.HasPrefix(s, "http://") {
+		return fmt.Sprintf("[%s](%s)", s, s)
+	}
+
+	return fmt.Sprintf("[%s](https://github.com/search?q=%s&type=code)", s, url.QueryEscape(s))
+}
+
 func (r Markdown) File(ctx context.Context, fr bincapz.FileReport) error {
 	markdownTable(ctx, &fr, r.w, tableConfig{Title: fmt.Sprintf("## %s [%s]", fr.Path, mdRisk(fr.RiskScore, fr.RiskLevel))})
 	return nil
@@ -129,16 +144,24 @@ func markdownTable(_ context.Context, fr *bincapz.FileReport, w io.Writer, rc ta
 		if found {
 			desc = before
 		}
+
+		if k.Behavior.ReferenceURL != "" {
+			desc = fmt.Sprintf("[%s](%s)", desc, k.Behavior.ReferenceURL)
+		}
+
 		if k.Behavior.RuleAuthor != "" {
+			author := k.Behavior.RuleAuthor
+			if k.Behavior.RuleAuthorURL != "" {
+				author = fmt.Sprintf("[%s](%s)", author, k.Behavior.RuleAuthorURL)
+			}
+
 			if desc != "" {
-				desc = fmt.Sprintf("%s, by %s", desc, k.Behavior.RuleAuthor)
+				desc = fmt.Sprintf("%s, by %s", desc, author)
 			} else {
-				desc = fmt.Sprintf("by %s", k.Behavior.RuleAuthor)
+				desc = fmt.Sprintf("by %s", author)
 			}
 		}
 
-		// lowercase first character for consistency
-		desc = strings.ToLower(string(desc[0])) + desc[1:]
 		risk := k.Behavior.RiskLevel
 		if k.Behavior.DiffAdded || rc.DiffAdded {
 			if rc.SkipAdded {
@@ -153,12 +176,16 @@ func markdownTable(_ context.Context, fr *bincapz.FileReport, w io.Writer, rc ta
 			risk = fmt.Sprintf("-%s", risk)
 		}
 
-		key := k.Key
+		key := fmt.Sprintf("[%s](%s)", k.Key, k.Behavior.RuleURL)
 		if strings.HasPrefix(risk, "+") {
 			key = fmt.Sprintf("**%s**", key)
 		}
 
-		evidence := strings.Join(k.Behavior.MatchStrings, "<br>")
+		matchLinks := []string{}
+		for _, m := range k.Behavior.MatchStrings {
+			matchLinks = append(matchLinks, matchFragmentLink(m))
+		}
+		evidence := strings.Join(matchLinks, "<br>")
 		data = append(data, []string{risk, key, desc, evidence})
 	}
 	table := tablewriter.NewWriter(w)

pkg/report/report.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -58,6 +59,9 @@ var dropRules = map[string]bool{
 	"3P/godmoderules/iddqd/god/mode": true,
 }
 
+// authorWithURLRe matcehs "Arnim Rupp (https://github.com/ruppde)"
+var authorWithURLRe = regexp.MustCompile(`(.*?) \((http.*)\)`)
+
 var dateRe = regexp.MustCompile(`[a-z]{3}\d{1,2}`)
 
 func yaraForgeKey(rule string) string {
@@ -91,9 +95,18 @@ func yaraForgeKey(rule string) string {
 	return strings.ReplaceAll(key, "signature/base", "signature_base")
 }
 
+// yaraForge returns whether the rule is sourced from YARAForge.
+func yaraForge(src string) bool {
+	return strings.Contains(src, "yara-rules")
+}
+
+func isValidURL(s string) bool {
+	_, err := url.Parse(s)
+	return err == nil
+}
+
 func generateKey(src string, rule string) string {
-	// It's Yara FORGE
-	if strings.Contains(src, "yara-rules") {
+	if yaraForge(src) {
 		return yaraForgeKey(rule)
 	}
 
@@ -109,6 +122,12 @@ func generateKey(src string, rule string) string {
 	return strings.ReplaceAll(key, ".yara", "")
 }
 
+func generateRuleURL(src string, rule string) string {
+	// Linking to exact commit and line number would be ideal, but
+	// we aren't parsing that information out of our YARA files yet
+	return fmt.Sprintf("https://github.com/chainguard-dev/bincapz/blob/main/rules/%s#%s", src, rule)
+}
+
 func ignoreMatch(tags []string, ignoreTags map[string]bool) bool {
 	for _, t := range tags {
 		if ignoreTags[t] {
@@ -229,6 +248,12 @@ func pathChecksum(path string) (string, error) {
 	return fmt.Sprintf("%x", h.Sum(nil)), nil
 }
 
+// fixURL fixes badly formed URLs.
+func fixURL(s string) string {
+	// YARAforge forgets to encode spaces, but encodes everything else
+	return strings.ReplaceAll(s, " ", "%20")
+}
+
 func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags []string, minScore int) (bincapz.FileReport, error) {
 	ignore := map[string]bool{}
 	for _, t := range ignoreTags {
@@ -250,9 +275,6 @@ func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags
 	pledges := []string{}
 	caps := []string{}
 	syscalls := []string{}
-	desc := ""
-	author := ""
-	license := ""
 	overallRiskScore := 0
 	riskCounts := map[int]int{}
 	packageRisks := []string{}
@@ -271,41 +293,66 @@ func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags
 			continue
 		}
 
+		ruleURL := generateRuleURL(m.Namespace, m.Rule)
 		packageRisks = append(packageRisks, key)
 
 		b := bincapz.Behavior{
 			RiskScore:    risk,
 			RiskLevel:    RiskLevels[risk],
 			MatchStrings: matchStrings(m.Rule, m.Strings),
+			RuleURL:      ruleURL,
 		}
 
 		for _, meta := range m.Metas {
-			switch meta.Identifier {
+			k := meta.Identifier
+			v := fmt.Sprintf("%s", meta.Value)
+			// Empty data is unusual, so just ignore it.
+			if k == "" || v == "" {
+				continue
+			}
+
+			switch k {
 			case "author":
-				author = fmt.Sprintf("%s", meta.Value)
-				if len(author) > len(b.RuleAuthor) {
-					b.RuleAuthor = author
-				}
-			case "license", "license_url":
-				license = fmt.Sprintf("%s", meta.Value)
-				if len(license) > len(b.RuleLicense) {
-					b.RuleLicense = license
+				b.RuleAuthor = v
+				m := authorWithURLRe.FindStringSubmatch(v)
+				if len(m) > 0 && isValidURL(m[2]) {
+					b.RuleAuthor = m[1]
+					b.RuleAuthorURL = m[2]
 				}
+			case "author_url":
+				b.RuleAuthorURL = v
+			case "license":
+				b.RuleLicense = v
+			case "license_url":
+				b.RuleLicenseURL = v
 			case "description", "threat_name", "name":
-				desc = fmt.Sprintf("%s", meta.Value)
-				if len(desc) > len(b.Description) {
-					b.Description = desc
+				if len(v) > len(b.Description) {
+					b.Description = v
 				}
+			case "ref", "reference":
+				u := fixURL(v)
+				if isValidURL(u) {
+					b.ReferenceURL = u
+				}
+			case "source_url":
+				// YARAforge forgets to encode spaces
+				b.RuleURL = fixURL(v)
 			case "pledge":
-				pledges = append(pledges, fmt.Sprintf("%s", meta.Value))
+				pledges = append(pledges, v)
 			case "syscall":
-				sy := strings.Split(fmt.Sprintf("%s", meta.Value), ",")
+				sy := strings.Split(v, ",")
 				syscalls = append(syscalls, sy...)
 			case "cap":
-				caps = append(caps, fmt.Sprintf("%s", meta.Value))
+				caps = append(caps, v)
 			}
 		}
 
+		// Fix YARA Forge rules that record their author URL as reference URLs
+		if strings.HasPrefix(b.RuleURL, b.ReferenceURL) {
+			b.RuleAuthorURL = b.ReferenceURL
+			b.ReferenceURL = ""
+		}
+
 		// Meta names are weird and unfortunate, depending if they hold a value
 		if strings.HasPrefix(key, "meta/") {
 			k := strings.ReplaceAll(filepath.Dir(key), "meta/", "")
@@ -321,22 +368,25 @@ func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags
 			continue
 		}
 
-		// We forgot :(
+		// If the rule does not have a description, make one up based on the rule name
 		if b.Description == "" {
 			b.Description = strings.ReplaceAll(m.Rule, "_", " ")
 		}
 
-		// We've already seen a similar behavior: do we augment it or replace it?
 		existing, exists := fr.Behaviors[key]
+		// If we have matched a lower priority rule in the same namespace, replace it
 		if !exists || existing.RiskScore < b.RiskScore {
 			fr.Behaviors[key] = b
 			continue
 		}
 
+		// If the existing description is longer,
 		if len(existing.Description) < len(b.Description) {
 			existing.Description = b.Description
 			fr.Behaviors[key] = existing
 		}
+
+		// TODO: If we match multiple rules within a single namespace, merge matchstrings
 	}
 	// If something has a lot of high, it's probably critical
 	if riskCounts[3] >= 4 {

rules/combo/backdoor/iptables.yara

@@ -1,7 +1,7 @@
 
 rule iptables_upload_http : notable {
 	meta:
-		description = "Uploads, uses iptables and HTTP"
+		description = "uploads, uses iptables and HTTP"
 	strings:
 		$ref1 = /upload[a-zA-Z]{0,16}/
 		$ref2 = "HTTP" fullword

rules/combo/backdoor/net_exec.yara

@@ -34,7 +34,7 @@ rule ssh_socks5_exec : notable {
 
 rule progname_socket_waitpid : suspicious {
 	meta:
-		description = "Sets program name, accesses internet, calls programs"
+		description = "sets process name, accesses internet, calls programs"
 	strings:
 		$dlsym = "__progname" fullword
 		$openpty = "socket" fullword

rules/combo/backdoor/net_term.yara

@@ -54,7 +54,7 @@ rule python_pty_spawner : suspicious {
 
 rule spectralblur_alike : suspicious {
 	meta:
-		description = "Uploads, provides a terminal, runs program"
+		description = "uploads, provides a terminal, runs program"
 	strings:
 		$upload = "upload"
 		$shell = "shell"
@@ -69,7 +69,7 @@ rule spectralblur_alike : suspicious {
 
 rule miner_kvryr_stak_alike : suspicious {
 	meta:
-		description = "Uploads, provides a terminal, runs program"
+		description = "uploads, provides a terminal, runs program"
 	strings:
 		$upload = "upload"
 		$shell = "shell"

rules/combo/stealer/upload-keychain-zip.yara

@@ -1,7 +1,7 @@
 
 rule previewers_alike: suspicious {
 	meta:
-		description = "Uploads, accesses a keychain, uses ZIP files"
+		description = "uploads, accesses a keychain, uses ZIP files"
 	strings:
 		$upload = "upload"
 		$zip = "zip"

rules/data/emdedded-app-manifest.yara

@@ -1,6 +1,7 @@
 rule app_manifest : notable {
 	meta:
 		description = "Contains embedded Microsoft Windows application manifest"
+		ref = "https://learn.microsoft.com/en-us/cpp/build/reference/manifestuac-embeds-uac-information-in-manifest?view=msvc-170"
 	strings:
 		$priv = "requestedPrivileges"
 		$exec = "requestedExecutionLevel"

rules/env/SHELL.yara

@@ -1,6 +1,6 @@
 rule SHELL {
   meta:
-	description = "users preferred SHELL path"
+	description = "path to active shell"
 	ref = "https://man.openbsd.org/login.1#ENVIRONMENT"
   strings:
 	$ref = "SHELL" fullword

rules/fs/node-create.yara

@@ -3,7 +3,8 @@ rule mknod {
 		pledge = "wpath"
 		syscall = "mknod"
 		capability = "CAP_MKNOD"
-		description = "able to make device files using mknod"
+		description = "create device files"
+		ref = "https://man7.org/linux/man-pages/man2/mknod.2.html"
 	strings:
 		$ref = "mknod" fullword
 	condition:
@@ -15,7 +16,8 @@ rule mknodat {
 		pledge = "wpath"
 		syscall = "mknodat"
 		capability = "CAP_MKNOD"
-		description = "able to make device files using mknod"
+		description = "create device files"
+		ref = "https://man7.org/linux/man-pages/man2/mknodat.2.html"
 	strings:
 		$ref2 = "mknodat" fullword
 	condition:

rules/kernel/hostname-get.yara

@@ -3,7 +3,7 @@ rule gethostname {
 	meta:
 		pledge = "sysctl"
 		syscall = "sysctl"
-		description = "gets the hostname of the machine"
+		description = "get computer host name"
 		ref = "https://man7.org/linux/man-pages/man2/sethostname.2.html"
 	strings:
 		$gethostname = "gethostname"