Cleanup ThreatHunting Keywords support (chainguard-dev#199)

* Cleanup ThreatHunting Keywords support * fix tests * add missing period
egibs · May 8, 2024 · 5727316 · 5727316
1 parent f97ca63
commit 5727316
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 26 deletions.
diff --git a/pkg/compile/compile.go b/pkg/compile/compile.go
@@ -19,7 +19,28 @@ var FS = rules.FS
 
 // badRules are noisy 3rd party rules to silently disable.
 var badRules = map[string]bool{
+	// YARAForge
 	"GODMODERULES_IDDQD_God_Mode_Rule": true,
+	// ThreatHunting Keywords (some duplicates)
+	"scp_greyware_tool_keyword":                  true,
+	"Antivirus_Signature_signature_keyword":      true,
+	"Dinjector_offensive_tool_keyword":           true,
+	"viperc2_offensive_tool_keyword":             true,
+	"github_greyware_tool_keyword":               true,
+	"wfuzz_offensive_tool_keyword":               true,
+	"nmap_greyware_tool_keyword":                 true,
+	"netcat_greyware_tool_keyword":               true,
+	"whoami_greyware_tool_keyword":               true,
+	"sftp_greyware_tool_keyword":                 true,
+	"empire_offensive_tool_keyword":              true,
+	"ssh_greyware_tool_keyword":                  true,
+	"wireshark_greyware_tool_keyword":            true,
+	"portscan_offensive_tool_keyword":            true,
+	"usbpcap_offensive_tool_keyword":             true,
+	"koadic_offensive_tool_keyword":              true,
+	"vsftpd_greyware_tool_keyword":               true,
+	"blackcat_ransomware_offensive_tool_keyword": true,
+	"mythic_offensive_tool_keyword":              true,
 }
 
 // rulesWithWarnings determines what to do with rules that have known warnings: true=keep, false=disable.

diff --git a/pkg/report/report.go b/pkg/report/report.go
@@ -139,15 +139,16 @@ func ignoreMatch(tags []string, ignoreTags map[string]bool) bool {
 	return false
 }
 
-func behaviorRisk(ns string, tags []string) int {
+func behaviorRisk(ns string, rule string, tags []string) int {
 	risk := 1
 
 	// default to critical
 	if thirdParty(ns) {
 		risk = 4
-		if strings.Contains(ns, "keyword") {
-			risk = 2
-		}
+	}
+
+	if strings.Contains(ns, "keyword") || strings.Contains(rule, "keyword") {
+		risk = 2
 	}
 
 	levels := map[string]int{
@@ -266,7 +267,7 @@ func mungeDescription(s string) string {
 	// out: references 'Nsight RMM'
 	m := threatHuntingKeywordRe.FindStringSubmatch(s)
 	if len(m) > 0 {
-		return fmt.Sprintf("references '%s'", m[1])
+		return fmt.Sprintf("references '%s' tool", m[1])
 	}
 	return s
 }
@@ -297,7 +298,7 @@ func Generate(ctx context.Context, path string, mrs yara.MatchRules, ignoreTags
 	packageRisks := []string{}
 
 	for _, m := range mrs {
-		risk := behaviorRisk(m.Namespace, m.Tags)
+		risk := behaviorRisk(m.Namespace, m.Rule, m.Tags)
 		if risk > overallRiskScore {
 			overallRiskScore = risk
 		}

diff --git a/samples/Linux/2023.FreeDownloadManager/freedownloadmanager.sdiff b/samples/Linux/2023.FreeDownloadManager/freedownloadmanager.sdiff
@@ -1,5 +1,5 @@
 *** changed: Linux/2023.FreeDownloadManager/freedownloadmanager_infected_postinst
-+3P/threat_hunting_keywords/touch
++3P/threat_hunting/touch
 +data/embedded/base64/terms
 +data/embedded/base64/url
 +encoding/base64

diff --git a/samples/Python/2022.PyPI.valyrian_debug/valyrian_debug_setup.py.simple b/samples/Python/2022.PyPI.valyrian_debug/valyrian_debug_setup.py.simple
@@ -1,5 +1,4 @@
 # Python/2022.PyPI.valyrian_debug/valyrian_debug_setup.py
-3P/threat_hunting_keywords/whoami
 combo/backdoor/py_setuptools
 combo/recon/system_network
 exec/pipe

diff --git a/samples/Windows/2024.GitHub.Clipper/main.exe.simple b/samples/Windows/2024.GitHub.Clipper/main.exe.simple
@@ -2,12 +2,10 @@
 3P/ditekshen/discordurl
 3P/ditekshen/rawgithub/url
 3P/ditekshen/vm/evasion/macaddrcomb
-3P/threat_hunting_keywords
-3P/threat_hunting_keywords/cstealer
-3P/threat_hunting_keywords/fentanyl
-3P/threat_hunting_keywords/github
-3P/threat_hunting_keywords/runascs
-3P/threat_hunting_keywords/wireshark
+3P/threat_hunting
+3P/threat_hunting/cstealer
+3P/threat_hunting/fentanyl
+3P/threat_hunting/runascs
 archives/zip
 combo/net/scan_tool
 combo/stealer/browser

diff --git a/samples/Windows/2024.Sharp/sharpil_RAT.exe.md b/samples/Windows/2024.Sharp/sharpil_RAT.exe.md
@@ -1,12 +1,12 @@
 ## Windows/2024.Sharp/sharpil_RAT.exe [🚨 CRITICAL]
 
-|   RISK   |                                                                                  KEY                                                                                  |                                                                                  DESCRIPTION                                                                                   |                                                                                   EVIDENCE                                                                                   |
-|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| CRITICAL | [3P/ditekshen/telegramchatbot](https://github.com/ditekshen/detection/blob/c37b067259715d4c93ac274a0830c54b355556a1/yara/indicator_suspicious.yar#L1291-L1306)        | Detects executables using Telegram Chat Bot, by [ditekSHen](https://github.com/ditekshen/detection)                                                                            | $p1<br>$p2<br>$s1<br>$s2<br>$s4                                                                                                                                              |
-| MEDIUM   | [3P/threat_hunting_keywords/telegram](https://github.com/chainguard-dev/bincapz/blob/main/rules/yara/threat_hunting_keywords/all.yara#telegram_greyware_tool_keyword) | [references 'telegram'](https://github.com/mthcht/ThreatHunting-Keywords), by @mthcht                                                                                          | $string1_telegram_greyware_tool_keyword                                                                                                                                      |
-| MEDIUM   | [data/emdedded/app/manifest](https://github.com/chainguard-dev/bincapz/blob/main/rules/data/emdedded-app-manifest.yara#app_manifest)                                  | [Contains embedded Microsoft Windows application manifest](https://learn.microsoft.com/en-us/cpp/build/reference/manifestuac-embeds-uac-information-in-manifest?view=msvc-170) | [requestedExecutionLevel](https://github.com/search?q=requestedExecutionLevel&type=code)<br>[requestedPrivileges](https://github.com/search?q=requestedPrivileges&type=code) |
-| MEDIUM   | [net/download](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/download.yara#download)                                                                  | download files                                                                                                                                                                 | [DownloadString](https://github.com/search?q=DownloadString&type=code)<br>[Downloads](https://github.com/search?q=Downloads&type=code)                                       |
-| MEDIUM   | [net/upload](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/upload.yara#upload)                                                                        | uploads files                                                                                                                                                                  | [UploadData](https://github.com/search?q=UploadData&type=code)                                                                                                               |
-| LOW      | [net/wireless](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/wireless.yara#bssid)                                                                     | wireless network base station ID                                                                                                                                               | [BSSID](https://github.com/search?q=BSSID&type=code)                                                                                                                         |
-| LOW      | [ref/words/password](https://github.com/chainguard-dev/bincapz/blob/main/rules/ref/words/password.yara#password)                                                      | references a 'password'                                                                                                                                                        | [Passwords](https://github.com/search?q=Passwords&type=code)                                                                                                                 |
+|   RISK   |                                                                              KEY                                                                               |                                                                                  DESCRIPTION                                                                                   |                                                                                   EVIDENCE                                                                                   |
+|----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| CRITICAL | [3P/ditekshen/telegramchatbot](https://github.com/ditekshen/detection/blob/c37b067259715d4c93ac274a0830c54b355556a1/yara/indicator_suspicious.yar#L1291-L1306) | Detects executables using Telegram Chat Bot, by [ditekSHen](https://github.com/ditekshen/detection)                                                                            | $p1<br>$p2<br>$s1<br>$s2<br>$s4                                                                                                                                              |
+| MEDIUM   | [3P/threat_hunting/telegram](https://github.com/chainguard-dev/bincapz/blob/main/rules/yara/threat_hunting/all.yara#telegram_greyware_tool_keyword)            | [references 'telegram' tool](https://github.com/mthcht/ThreatHunting-Keywords), by @mthcht                                                                                     | $string1_telegram_greyware_tool_keyword                                                                                                                                      |
+| MEDIUM   | [data/emdedded/app/manifest](https://github.com/chainguard-dev/bincapz/blob/main/rules/data/emdedded-app-manifest.yara#app_manifest)                           | [Contains embedded Microsoft Windows application manifest](https://learn.microsoft.com/en-us/cpp/build/reference/manifestuac-embeds-uac-information-in-manifest?view=msvc-170) | [requestedExecutionLevel](https://github.com/search?q=requestedExecutionLevel&type=code)<br>[requestedPrivileges](https://github.com/search?q=requestedPrivileges&type=code) |
+| MEDIUM   | [net/download](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/download.yara#download)                                                           | download files                                                                                                                                                                 | [DownloadString](https://github.com/search?q=DownloadString&type=code)<br>[Downloads](https://github.com/search?q=Downloads&type=code)                                       |
+| MEDIUM   | [net/upload](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/upload.yara#upload)                                                                 | uploads files                                                                                                                                                                  | [UploadData](https://github.com/search?q=UploadData&type=code)                                                                                                               |
+| LOW      | [net/wireless](https://github.com/chainguard-dev/bincapz/blob/main/rules/net/wireless.yara#bssid)                                                              | wireless network base station ID                                                                                                                                               | [BSSID](https://github.com/search?q=BSSID&type=code)                                                                                                                         |
+| LOW      | [ref/words/password](https://github.com/chainguard-dev/bincapz/blob/main/rules/ref/words/password.yara#password)                                               | references a 'password'                                                                                                                                                        | [Passwords](https://github.com/search?q=Passwords&type=code)                                                                                                                 |
 
diff --git a/samples/samples_test.go b/samples/samples_test.go
@@ -10,6 +10,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"testing"
 
@@ -224,6 +225,16 @@ func TestDiff(t *testing.T) {
 	}
 }
 
+// reduceMarkdown reduces markdown output to simply diff output.
+func reduceMarkdown(s string) string {
+	spaceRe := regexp.MustCompile(` +`)
+	dashRe := regexp.MustCompile(` -`)
+
+	s = spaceRe.ReplaceAllString(s, " ")
+	s = dashRe.ReplaceAllString(s, " ")
+	return s
+}
+
 func TestMarkdown(t *testing.T) {
 	ctx := slogtest.TestContextWithLogger(t)
 	clog.FromContext(ctx).With("test", "TestMarkDown")
@@ -253,7 +264,7 @@ func TestMarkdown(t *testing.T) {
 				t.Fatalf("testdata read failed: %v", err)
 			}
 
-			want := string(td)
+			want := reduceMarkdown(string(td))
 			var out bytes.Buffer
 			simple, err := render.New("markdown", &out)
 			if err != nil {
@@ -280,7 +291,7 @@ func TestMarkdown(t *testing.T) {
 				t.Fatalf("full: %v", err)
 			}
 
-			got := out.String()
+			got := reduceMarkdown(out.String())
 			if diff := cmp.Diff(want, got); diff != "" {
 				t.Errorf("markdown output mismatch: (-want +got):\n%s", diff)
 			}

diff --git a/...arty/yara/threat_hunting_keywords/RELEASE → third_party/yara/threat_hunting/RELEASE b/...arty/yara/threat_hunting_keywords/RELEASE → third_party/yara/threat_hunting/RELEASE
diff --git a/...rty/yara/threat_hunting_keywords/all.yara → third_party/yara/threat_hunting/all.yara b/...rty/yara/threat_hunting_keywords/all.yara → third_party/yara/threat_hunting/all.yara
diff --git a/third_party/yara/update.sh b/third_party/yara/update.sh
@@ -44,7 +44,7 @@ function update_dep() {
 		popd || exit 1
 		find "${tmpdir}" \( -name "*.yar*" -o -name "*LICENSE*" \) -print -exec cp {} "${kind}" \;
 		;;
-	threat_hunting_keywords)
+	threat_hunting)
 		rel=$(latest_github_release mthcht/ThreatHunting-Keywords-yara-rules)
 		curl -L -o "${tmpdir}/keywords.zip" "https://github.com/mthcht/ThreatHunting-Keywords-yara-rules/archive/refs/tags/${rel}.zip"
 		unzip -o -j "${tmpdir}/keywords.zip" ThreatHunting-Keywords-yara-rules-1.0.1/yara_rules/all.yara -d "${kind}"