[FEATURE] Filter out hardcoded patterns, generate tag only for custom…

… ones

pe_sieve_ver_short.h

@@ -3,6 +3,6 @@
 #define PESIEVE_MAJOR_VERSION 0
 #define PESIEVE_MINOR_VERSION 3
 #define PESIEVE_MICRO_VERSION 8
-#define PESIEVE_PATCH_VERSION 6
+#define PESIEVE_PATCH_VERSION 7
 
-#define PESIEVE_VERSION_STR	"0.3.8.6"
+#define PESIEVE_VERSION_STR	"0.3.8.7"

scanners/workingset_scanner.cpp

@@ -44,7 +44,7 @@ namespace pesieve {
 
 size_t WorkingSetScanReport::generateTags(const std::string& reportPath)
 {
-	if (matched_patterns.size() == 0) {
+	if (this->custom_matched.size() == 0) {
 		return 0;
 	}
 	std::ofstream patch_report;
@@ -53,7 +53,7 @@ size_t WorkingSetScanReport::generateTags(const std::string& reportPath)
 		return 0;
 	}
 	size_t count = 0;
-	for (auto itr = matched_patterns.begin(); itr != matched_patterns.end(); itr++) {
+	for (auto itr = custom_matched.begin(); itr != custom_matched.end(); ++itr) {
 		sig_finder::Match m = *itr;
 		if (match_to_tag(patch_report, ';', this->match_area_start, m)) count++;
 	}
@@ -80,9 +80,13 @@ bool pesieve::WorkingSetScanner::checkAreaContent(IN MemPageData& memPage, OUT W
 	bool codeS = false;
 	bool obfuscated = false;
 
+	size_t custom_matched_count = 0;
+
 	if (matcher::is_matcher_ready()) {
-		const size_t matches_count = matcher::find_all_patterns(memPage.getLoadedData(noPadding), memPage.getLoadedSize(noPadding), my_report->matched_patterns);
-		if (matches_count) {
+		std::vector<sig_finder::Match> allMatched;
+		my_report->all_matched_count = matcher::find_all_patterns(memPage.getLoadedData(noPadding), memPage.getLoadedSize(noPadding), allMatched);
+		custom_matched_count = matcher::filter_custom(allMatched, my_report->custom_matched);
+		if (my_report->all_matched_count) {
 			my_report->match_area_start = memPage.getStartOffset(noPadding);
 			codeP = true;
 			code = true;
@@ -141,7 +145,7 @@ bool pesieve::WorkingSetScanner::checkAreaContent(IN MemPageData& memPage, OUT W
 	}
 	my_report->has_shellcode = code;
 
-	if (codeP && this->args.pattern_file.length) {
+	if (custom_matched_count && this->args.pattern_file.length) {
 		my_report->has_patterns = true;
 		my_report->status = SCAN_SUSPICIOUS;
 	}

scanners/workingset_scanner.h

@@ -38,7 +38,7 @@ namespace pesieve {
 			has_shellcode = true;
 			mapping_type = 0;
 			match_area_start = 0;
-			has_patterns = false;
+			all_matched_count = 0;
 		}
 
 		const virtual bool toJSON(std::stringstream &outs, size_t level, const pesieve::t_json_level &jdetails)
@@ -59,10 +59,15 @@ namespace pesieve {
 			outs << ",\n";
 			OUT_PADDED(outs, level, "\"has_shellcode\" : ");
 			outs << std::dec << has_shellcode;
-			if (matched_patterns.size()) {
+			if (all_matched_count) {
 				outs << ",\n";
-				OUT_PADDED(outs, level, "\"matched_patterns\" : ");
-				outs << std::dec << matched_patterns.size();
+				OUT_PADDED(outs, level, "\"patterns_matched\" : ");
+				outs << std::dec << all_matched_count;
+			}
+			if (custom_matched.size()) {
+				outs << ",\n";
+				OUT_PADDED(outs, level, "\"custom_matched\" : ");
+				outs << std::dec << custom_matched.size();
 			}
 			if (!is_executable) {
 				outs << ",\n";
@@ -103,7 +108,8 @@ namespace pesieve {
 		bool has_shellcode;
 		bool has_patterns;
 		util::ByteBuffer data_cache;
-		std::vector<sig_finder::Match> matched_patterns;
+		std::vector<sig_finder::Match> custom_matched;
+		size_t all_matched_count;
 		size_t match_area_start;
 #ifdef CALC_PAGE_STATS
 		AreaMultiStats stats;

utils/artefacts_util.cpp

@@ -20,6 +20,8 @@ BYTE* pesieve::util::find_pattern(BYTE* buffer, size_t buf_size, BYTE* pattern_b
 
 namespace pesieve {
 
+	std::set<DWORD> HardcodedPatterns;
+
 	size_t init_32_patterns(Node* rootN)
 	{
 		if (!rootN) return 0;
@@ -29,7 +31,11 @@ namespace pesieve {
 		{
 			const t_pattern& pattern = patterns32[i];
 			std::string name = "prolog32_" + std::to_string(i);
-			if (rootN->addPattern(name.c_str(), pattern.ptr, pattern.size)) added++;
+			Signature sign(name, pattern.ptr, pattern.size);
+			if (rootN->addPattern(sign)) {
+				HardcodedPatterns.insert(sign.checksum());
+				added++;
+			}
 		}
 		return added;
 	}
@@ -43,7 +49,11 @@ namespace pesieve {
 		{
 			const t_pattern &pattern = patterns64[i];
 			std::string name = "prolog64_" + std::to_string(i);
-			if (rootN->addPattern(name.c_str(), pattern.ptr, pattern.size)) added++;
+			Signature sign(name, pattern.ptr, pattern.size);
+			if (rootN->addPattern(sign)) {
+				HardcodedPatterns.insert(sign.checksum());
+				added++;
+			}
 		}
 		return added;
 	}
@@ -171,3 +181,20 @@ size_t pesieve::matcher::find_all_patterns(BYTE* loadedData, size_t loadedSize,
 	const size_t matches =  sig_finder::find_all_matches(mainMatcher, loadedData, loadedSize, allMatches);
 	return matches;
 }
+
+size_t pesieve::matcher::filter_custom(std::vector<sig_finder::Match>& allMatches, std::vector<sig_finder::Match>& customPatternMatches)
+{
+	size_t customCount = 0;
+	for (auto itr = allMatches.begin(); itr != allMatches.end(); ++itr) {
+		sig_finder::Match m = *itr;
+		if (m.sign) {
+			const DWORD checks = m.sign->checksum();
+			if (HardcodedPatterns.find(checks) != HardcodedPatterns.end()) {
+				continue;
+			}
+			customPatternMatches.push_back(m);
+			customCount++;
+		}
+	}
+	return customCount;
+}

utils/artefacts_util.h

@@ -46,5 +46,7 @@ namespace pesieve {
 
 		size_t find_all_patterns(BYTE* loadedData, size_t loadedSize, std::vector<sig_finder::Match>& allMatches);
 
+		size_t filter_custom(std::vector<sig_finder::Match>& allMatches, std::vector<sig_finder::Match>& customPatternMatches);
+
 	}; //namespace matcher
 }