Merge branch 'pattern_report2'

include/pe_sieve_types.h

@@ -130,6 +130,7 @@ namespace pesieve {
 		t_json_level json_lvl;  ///< level of the details of the JSON report
 		char output_dir[MAX_PATH + 1];  ///< the root directory where the output should be saved (default: current directory)
 		PARAM_STRING modules_ignored; ///< a list of modules that will not be scanned, separated by PARAM_LIST_SEPARATOR
+		PARAM_STRING pattern_file; ///< a file with additional patterns for code recognition
 	} t_params;
 
 	//!  Final summary about the scanned process.

params.h

@@ -30,6 +30,7 @@ using namespace pesieve;
 #define PARAM_JSON_LVL "jlvl"
 #define PARAM_DIR "dir"
 #define PARAM_MINIDUMP "minidmp"
+#define PARAM_PATTERN "pattern"
 
 
 bool alloc_strparam(PARAM_STRING& strparam, ULONG len)
@@ -120,6 +121,9 @@ class PEsieveParams : public Params
 				enumParam->addEnumValue(mode, shellc_mode_mode_to_id(mode), translate_shellc_mode(mode));
 			}
 		}
+
+		this->addParam(new StringParam(PARAM_PATTERN, false));
+		this->setInfo(PARAM_PATTERN, "Set additional shellcode patterns (file in the SIG format).");
 
 		//PARAM_OBFUSCATED
 		enumParam = new EnumParam(PARAM_OBFUSCATED, "obfusc_mode", false);
@@ -211,6 +215,7 @@ class PEsieveParams : public Params
 		this->addParamToGroup(PARAM_SHELLCODE, str_group);
 		this->addParamToGroup(PARAM_OBFUSCATED, str_group);
 		this->addParamToGroup(PARAM_THREADS, str_group);
+		this->addParamToGroup(PARAM_PATTERN, str_group);
 
 		str_group = "4. dump options";
 		this->addGroup(new ParamGroup(str_group));
@@ -268,6 +273,7 @@ class PEsieveParams : public Params
 		copyVal<EnumParam>(PARAM_DUMP_MODE, ps.dump_mode);
 
 		copyCStr<StringParam>(PARAM_DIR, ps.output_dir, _countof(ps.output_dir));
+		fillStringParam(PARAM_PATTERN, ps.pattern_file);
 	}
 
 	void printBanner()

pe_sieve.cpp

@@ -14,6 +14,8 @@
 #include "utils/console_color.h"
 #include "color_scheme.h"
 
+#include "utils/artefacts_util.h"
+
 using namespace pesieve;
 using namespace pesieve::util;
 
@@ -173,6 +175,21 @@ namespace pesieve {
 }; //namespace pesieve
 
 
+namespace pesieve {
+
+	inline bool is_by_patterns(const t_shellc_mode& shellc_mode)
+	{
+		switch (shellc_mode) {
+		case  SHELLC_PATTERNS:
+		case  SHELLC_PATTERNS_OR_STATS:
+		case SHELLC_PATTERNS_AND_STATS:
+			return true;
+		}
+		return false;
+	}
+
+}; // namespace pesieve
+
 pesieve::ReportEx* pesieve::scan_and_dump(IN const pesieve::t_params args)
 {
 	ReportEx *report = new(std::nothrow) ReportEx();
@@ -187,6 +204,17 @@ pesieve::ReportEx* pesieve::scan_and_dump(IN const pesieve::t_params args)
 		if (!args.quiet) std::cerr << "[-] Could not set debug privilege" << std::endl;
 	}
 
+	if (args.pattern_file.length) {
+		size_t loaded = matcher::load_pattern_file(args.pattern_file.buffer);
+		if (!args.quiet) {
+			if (loaded) std::cout << "[+] Pattern file loaded: " << args.pattern_file.buffer << ", Signs: " << loaded << std::endl;
+			else std::cerr << "[-] Failed to load pattern file: " << args.pattern_file.buffer << std::endl;
+		}
+	}
+	if (is_by_patterns(args.shellcode)) {
+		matcher::init_shellcode_patterns();
+	}
+
 	try {
 		orig_proc = open_process(args.pid, args.make_reflection, args.quiet);
 		HANDLE target_proc = orig_proc;
@@ -217,8 +245,8 @@ pesieve::ReportEx* pesieve::scan_and_dump(IN const pesieve::t_params args)
 		ProcessScanner scanner(target_proc, is_reflection, args);
 		report->scan_report = scanner.scanRemote();
 
-		// dump process
 		if (report->scan_report) {
+			// dump elements from the process:
 			report->dump_report = make_dump(target_proc, is_reflection, args, *report->scan_report);
 		}
 	}

pe_sieve_ver_short.h

@@ -3,6 +3,6 @@
 #define PESIEVE_MAJOR_VERSION 0
 #define PESIEVE_MINOR_VERSION 3
 #define PESIEVE_MICRO_VERSION 8
-#define PESIEVE_PATCH_VERSION 5
+#define PESIEVE_PATCH_VERSION 6
 
 #define PESIEVE_VERSION_STR	"0.3.8.6"

postprocessors/dump_report.cpp

@@ -14,9 +14,13 @@ const bool pesieve::ModuleDumpReport::toJSON(std::stringstream &outs, size_t lev
 		OUT_PADDED(outs, level, "\"dump_file\" : ");
 		outs << "\"" << peconv::get_file_name(dumpFileName) << "\"" << ",\n";
 	}
-	if (tagsFileName.length()) {
+	if (hooksTagFileName.length()) {
 		OUT_PADDED(outs, level, "\"tags_file\" : ");
-		outs << "\"" << peconv::get_file_name(tagsFileName) << "\"" << ",\n";
+		outs << "\"" << peconv::get_file_name(hooksTagFileName) << "\"" << ",\n";
+	}
+	if (patternsTagFileName.length()) {
+		OUT_PADDED(outs, level, "\"pattern_tags_file\" : ");
+		outs << "\"" << peconv::get_file_name(patternsTagFileName) << "\"" << ",\n";
 	}
 	if (impListFileName.length()) {
 		OUT_PADDED(outs, level, "\"imports_file\" : ");

postprocessors/dump_report.h

@@ -36,7 +36,8 @@ namespace pesieve {
 		bool isDumped;
 		std::string mode_info;
 		std::string dumpFileName;
-		std::string tagsFileName;
+		std::string hooksTagFileName;
+		std::string patternsTagFileName;
 		std::string impListFileName;
 		std::string notRecoveredFileName;
 		std::string iatHooksFileName;

postprocessors/results_dumper.cpp

@@ -359,11 +359,22 @@ bool pesieve::ResultsDumper::dumpModule(IN HANDLE processHandle,
 		std::string tags_file = modDumpReport->dumpFileName + ".tag";
 
 		if (codeScanReport->generateTags(tags_file)) {
-			modDumpReport->tagsFileName = tags_file;
+			modDumpReport->hooksTagFileName = tags_file;
 			modDumpReport->isReportDumped = true;
 		}
 	}
 
+	pesieve::WorkingSetScanReport* wsScanReport = dynamic_cast<pesieve::WorkingSetScanReport*>(mod);
+	if (wsScanReport) {
+		std::string tags_file = modDumpReport->dumpFileName + ".pattern.tag";
+
+		if (wsScanReport->generateTags(tags_file)) {
+			modDumpReport->patternsTagFileName = tags_file;
+			modDumpReport->isReportDumped = true;
+		}
+	}
+
+
 	IATScanReport* iatHooksReport = dynamic_cast<IATScanReport*>(mod);
 	if (iatHooksReport) {
 		std::string imports_not_rec_file = modDumpReport->dumpFileName + ".iat_hooks.txt";

scanners/mempage_data.h

@@ -30,6 +30,7 @@ namespace pesieve {
 		bool isInfoFilled() { return is_info_filled; }
 		size_t getLoadedSize(bool trimmed = false) { return loadedData.getDataSize(trimmed); }
 		const PBYTE getLoadedData(bool trimmed = false) { return (PBYTE)loadedData.getData(trimmed); }
+		const size_t getStartOffset(bool trimmed = false) { return loadedData.getStartOffset(trimmed); }
 
 		bool validatePtr(const LPVOID field_bgn, size_t field_size)
 		{

scanners/scan_report.cpp

@@ -1,5 +1,5 @@
 #include "scan_report.h"
-
+#include "../pe_sieve_ver_short.h"
 #include "headers_scanner.h"
 #include "code_scanner.h"
 #include "iat_scanner.h"
@@ -230,6 +230,8 @@ const bool pesieve::ProcessScanReport::toJSON(
 	stream << escape_path_separators(this->mainImagePath) << "\",\n";
 	OUT_PADDED(stream, level, "\"used_reflection\" : ");
 	stream << std::dec << report.is_reflection << ",\n";
+	OUT_PADDED(stream, level, "\"scanner_version\" : ");
+	stream << "\"" << PESIEVE_VERSION_STR << "\",\n";
 	OUT_PADDED(stream, level, "\"scanned\" : \n");
 	OUT_PADDED(stream, level, "{\n");
 	//stream << " {\n";

scanners/workingset_scanner.cpp

@@ -7,13 +7,15 @@
 #include "../utils/workingset_enum.h"
 #include "../utils/artefacts_util.h"
 
+#include <fstream>
+
 using namespace pesieve;
 using namespace pesieve::util;
 
 
 namespace pesieve {
 
-	bool is_by_stats(const t_shellc_mode& shellc_mode)
+	inline bool is_by_stats(const t_shellc_mode& shellc_mode)
 	{
 		switch (shellc_mode) {
 		case  SHELLC_STATS:
@@ -23,18 +25,42 @@ namespace pesieve {
 		}
 		return false;
 	}
+};
 
-	bool is_by_patterns(const t_shellc_mode& shellc_mode)
-	{
-		switch (shellc_mode) {
-		case  SHELLC_PATTERNS:
-		case  SHELLC_PATTERNS_OR_STATS:
-		case SHELLC_PATTERNS_AND_STATS:
-			return true;
-		}
-		return false;
+
+inline bool match_toTAG(std::ofstream& patch_report, const char delimiter, size_t start_offset, sig_finder::Match match)
+{
+	if (patch_report.is_open() && match.sign) {
+		patch_report << std::hex << match.offset + start_offset;
+		patch_report << delimiter;
+		patch_report << match.sign->name;
+		patch_report << delimiter;
+		patch_report << match.sign->size();
+		patch_report << std::endl;
+		return true;
 	}
-};
+	return false;
+}
+size_t WorkingSetScanReport::generateTags(const std::string& reportPath)
+{
+	if (matched_patterns.size() == 0) {
+		return 0;
+	}
+	std::ofstream patch_report;
+	patch_report.open(reportPath);
+	if (patch_report.is_open() == false) {
+		return 0;
+	}
+	size_t count = 0;
+	for (auto itr = matched_patterns.begin(); itr != matched_patterns.end(); itr++) {
+		sig_finder::Match m = *itr;
+		if (match_toTAG(patch_report, ';', this->match_area_start, m)) count++;
+	}
+	if (patch_report.is_open()) {
+		patch_report.close();
+	}
+	return count;
+}
 
 bool pesieve::WorkingSetScanner::checkAreaContent(IN MemPageData& memPage, OUT WorkingSetScanReport* my_report)
 {
@@ -45,14 +71,16 @@ bool pesieve::WorkingSetScanner::checkAreaContent(IN MemPageData& memPage, OUT W
 	const bool noPadding = true;
 
 	bool isByStats = is_by_stats(this->args.shellcode);
-	bool isByPatterns = is_by_patterns(this->args.shellcode);
 
 	bool code = false;
 	bool codeP = false;
 	bool codeS = false;
 	bool obfuscated = false;
-	if (isByPatterns) {
-		if (is_code(memPage.getLoadedData(noPadding), memPage.getLoadedSize(noPadding))) {
+
+	if (matcher::is_matcher_ready()) {
+		const size_t matches_count = matcher::find_all_patterns(memPage.getLoadedData(noPadding), memPage.getLoadedSize(noPadding), my_report->matched_patterns);
+		if (matches_count) {
+			my_report->match_area_start = memPage.getStartOffset(noPadding);
 			codeP = true;
 			code = true;
 			if (this->args.shellcode == SHELLC_PATTERNS_OR_STATS) {
@@ -110,8 +138,14 @@ bool pesieve::WorkingSetScanner::checkAreaContent(IN MemPageData& memPage, OUT W
 	}
 	my_report->has_shellcode = code;
 
+	if (codeP && this->args.pattern_file.length) {
+		my_report->has_patterns = true;
+		my_report->status = SCAN_SUSPICIOUS;
+	}
 	if ( (this->args.obfuscated != OBFUSC_NONE && obfuscated) || ((this->args.shellcode != SHELLC_NONE) && code) ){
 		my_report->status = SCAN_SUSPICIOUS;
+	}
+	if (my_report->status == SCAN_SUSPICIOUS) {
 		my_report->data_cache = memPage.loadedData;
 	}
 	return true;
@@ -172,8 +206,10 @@ WorkingSetScanReport* pesieve::WorkingSetScanner::scanExecutableArea(MemPageData
 			return my_report1;
 		}
 	}
-	if ((this->args.shellcode == SHELLC_NONE) && (this->args.obfuscated == OBFUSC_NONE)) {
-		// not a PE file, and we are not interested in shellcode or obfuscated contents, so just finish it here
+	if ((!matcher::is_matcher_ready())
+		&& (this->args.obfuscated == OBFUSC_NONE))
+	{
+		// not a PE file, and we are not interested in patterns or obfuscated contents, so just finish it here
 		return nullptr;
 	}
 

scanners/workingset_scanner.h

@@ -20,6 +20,8 @@
 #include "../stats/stats_analyzer.h"
 #endif
 
+#include <sig_finder.h>
+
 namespace pesieve {
 
 	//!  A report from the working set scan, generated by WorkingSetScanner
@@ -35,6 +37,8 @@ namespace pesieve {
 			has_pe = false; //not a PE file
 			has_shellcode = true;
 			mapping_type = 0;
+			match_area_start = 0;
+			has_patterns = false;
 		}
 
 		const virtual bool toJSON(std::stringstream &outs, size_t level, const pesieve::t_json_level &jdetails)
@@ -55,6 +59,11 @@ namespace pesieve {
 			outs << ",\n";
 			OUT_PADDED(outs, level, "\"has_shellcode\" : ");
 			outs << std::dec << has_shellcode;
+			if (matched_patterns.size()) {
+				outs << ",\n";
+				OUT_PADDED(outs, level, "\"matched_patterns\" : ");
+				outs << std::dec << matched_patterns.size();
+			}
 			if (!is_executable) {
 				outs << ",\n";
 				OUT_PADDED(outs, level, "\"is_executable\" : ");
@@ -86,11 +95,16 @@ namespace pesieve {
 #endif
 		}
 
+		size_t generateTags(const std::string &reportPath);
+
 		bool is_executable;
 		bool is_listed_module;
 		bool has_pe;
 		bool has_shellcode;
+		bool has_patterns;
 		util::ByteBuffer data_cache;
+		std::vector<sig_finder::Match> matched_patterns;
+		size_t match_area_start;
 #ifdef CALC_PAGE_STATS
 		AreaMultiStats stats;
 		AreaInfo area_info;