Merge pull request #9994 from nextcloud/feat/link-checking

Feat: link checking for phishing detection

composer.json

@@ -27,6 +27,7 @@
 		"bytestream/horde-util": "^2.7.0",
 		"cerdic/css-tidy": "v2.1.0",
 		"ezyang/htmlpurifier": "4.17.0",
+		"glenscott/url-normalizer": "^1.4",
 		"gravatarphp/gravatar": "dev-master#6b9f6a45477ce48285738d9d0c3f0dbf97abe263",
 		"hamza221/html2text": "^1.0",
 		"jeremykendall/php-domain-parser": "^6.3",

lib/Service/PhishingDetection/LinkCheck.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\Service\PhishingDetection;
+
+use OCA\Mail\PhishingDetectionResult;
+use OCP\IL10N;
+use URL\Normalizer;
+
+class LinkCheck {
+	protected IL10N $l10n;
+
+
+	public function __construct(IL10N $l10n) {
+		$this->l10n = $l10n;
+	}
+	// checks if link text is meant to look like a link
+	private function textLooksLikeALink(string $text): bool {
+		// based on https://gist.github.com/gruber/8891611
+		$pattern = '/(?i)\b((?:https?:(?:\/{1,3}|[a-z0-9%])|[a-z0-9.\-]+[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\/)(?:[^\s()<>{}\[\]]+|\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\))+(?:\([^\s()]*?\([^\s()]+\)[^\s()]*?\)|\([^\s]+?\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’])|(?:(?<!@)[a-z0-9]+(?:[.\-][a-z0-9]+)*[.](?:com|net|org|edu|gov|mil|aero|asia|biz|cat|coop|info|int|jobs|mobi|museum|name|post|pro|tel|travel|xxx|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cs|cu|cv|cx|cy|cz|dd|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|Ja|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw)\b\/?(?!@)))/';
+
+		return preg_match($pattern, $text) === 1;
+	}
+
+	private function getInnerText(\DOMElement $node) : string {
+		$innerText = '';
+		foreach ($node->childNodes as $child) {
+			if ($child->nodeType === XML_TEXT_NODE) {
+				$innerText .= $child->nodeValue;
+			} elseif ($child->nodeType === XML_ELEMENT_NODE) {
+				$innerText .= $this->getInnerText($child);
+			}
+		}
+		return $innerText;
+	}
+
+	public function run(string $htmlMessage) : PhishingDetectionResult {
+
+		$results = [];
+		$zippedArray = [];
+
+		$dom = new \DOMDocument();
+		libxml_use_internal_errors(true);
+		$dom->loadHTML($htmlMessage);
+		libxml_use_internal_errors();
+		$anchors = $dom->getElementsByTagName('a');
+		foreach ($anchors as $anchor) {
+			$href = $anchor->getAttribute('href');
+			$linkText = $this->getInnerText($anchor);
+			$zippedArray[] = [
+				'href' => $href,
+				'linkText' => $linkText
+			];
+		}
+		foreach ($zippedArray as $zipped) {
+			$un = new Normalizer($zipped['href']);
+			$url = $un->normalize();
+			if($this->textLooksLikeALink($zipped['linkText'])) {
+				if(parse_url($url, PHP_URL_HOST) !== parse_url($zipped['linkText'], PHP_URL_HOST)) {
+					$results[] = [
+						'href' => $url,
+						'linkText' => $zipped['linkText'],
+					];
+				}
+			}
+		}
+		if(count($results) > 0) {
+			return new PhishingDetectionResult(PhishingDetectionResult::LINK_CHECK, true, $this->l10n->t('Some addresses in this message are not matching the link text'), $results);
+		}
+		return  new PhishingDetectionResult(PhishingDetectionResult::LINK_CHECK, false);
+
+	}
+
+}

lib/Service/PhishingDetection/PhishingDetectionService.php

@@ -14,11 +14,12 @@
 use OCA\Mail\PhishingDetectionList;
 
 class PhishingDetectionService {
-	public function __construct(private ContactCheck $contactCheck, private CustomEmailCheck $customEmailCheck, private DateCheck $dateCheck, private ReplyToCheck $replyToCheck) {
+	public function __construct(private ContactCheck $contactCheck, private CustomEmailCheck $customEmailCheck, private DateCheck $dateCheck, private ReplyToCheck $replyToCheck, private LinkCheck $linkCheck) {
 		$this->contactCheck = $contactCheck;
 		$this->customEmailCheck = $customEmailCheck;
 		$this->dateCheck = $dateCheck;
 		$this->replyToCheck = $replyToCheck;
+		$this->linkCheck = $linkCheck;
 	}
 
 
@@ -38,6 +39,9 @@ public function checkHeadersForPhishing(Horde_Mime_Headers $headers, bool $hasHt
 		$list->addCheck($this->contactCheck->run($fromFN, $fromEmail));
 		$list->addCheck($this->dateCheck->run($date));
 		$list->addCheck($this->customEmailCheck->run($fromEmail, $customEmail));
+		if ($hasHtmlMessage) {
+			$list->addCheck($this->linkCheck->run($htmlMessage));
+		}
 		return $list->jsonSerialize();
 	}
 }

src/components/PhishingWarning.vue

@@ -11,16 +11,28 @@
 		<ul v-for="(warning,index) in warnings" :key="index" class="warning__list">
 			<li>{{ warning.message }}</li>
 		</ul>
+		<div v-if="linkWarning !== undefined" class="warning__links">
+			<NcButton class="warning__links__button" type="Tertiary" @click="showMore = !showMore">
+				{{ showMore? t('mail','hide suspicious links') :t('mail','Show suspicious links') }}
+			</NcButton>
+			<div v-if="showMore">
+				<ul v-for="(link,index) in linkWarning.additionalData" :key="index" class="warning__list">
+					<li><b>href: </b>{{ link.href }} : <b>{{ t('mail','link text') }}</b> {{ link.linkText }} </li>
+				</ul>
+			</div>
+		</div>
 	</div>
 </template>
 <script>
 import IconAlertOutline from 'vue-material-design-icons/AlertOutline.vue'
+import { NcButton } from '@nextcloud/vue'
 
 export default {
 
 	name: 'PhishingWarning',
 	components: {
 		IconAlertOutline,
+		NcButton,
 	},
 	props: {
 		phishingData: {
@@ -37,6 +49,9 @@ export default {
 		warnings() {
 			return this.phishingData.filter(check => check.isPhishing)
 		},
+		linkWarning() {
+			return this.phishingData.find(check => check.type === 'Link')
+		},
 	},
 
 }

tests/Integration/Service/Phishing/PhishingDetectionServiceIntegrationTest.php

@@ -14,6 +14,7 @@
 use OCA\Mail\Service\PhishingDetection\ContactCheck;
 use OCA\Mail\Service\PhishingDetection\CustomEmailCheck;
 use OCA\Mail\Service\PhishingDetection\DateCheck;
+use OCA\Mail\Service\PhishingDetection\LinkCheck;
 use OCA\Mail\Service\PhishingDetection\PhishingDetectionService;
 use OCA\Mail\Service\PhishingDetection\ReplyToCheck;
 use OCP\AppFramework\Utility\ITimeFactory;
@@ -30,6 +31,7 @@ class PhishingDetectionServiceIntegrationTest extends TestCase {
 	private CustomEmailCheck $customEmailCheck;
 	private DateCheck $dateCheck;
 	private ReplyToCheck $replyToCheck;
+	private LinkCheck $linkCheck;
 	private PhishingDetectionService $service;
 
 	protected function setUp(): void {
@@ -40,7 +42,8 @@ protected function setUp(): void {
 		$this->customEmailCheck = new CustomEmailCheck($this->l10n);
 		$this->dateCheck = new DateCheck($this->l10n, \OC::$server->get(ITimeFactory::class));
 		$this->replyToCheck = new ReplyToCheck($this->l10n);
-		$this->service = new PhishingDetectionService($this->contactCheck, $this->customEmailCheck, $this->dateCheck, $this->replyToCheck);
+		$this->linkCheck = new LinkCheck($this->l10n);
+		$this->service = new PhishingDetectionService($this->contactCheck, $this->customEmailCheck, $this->dateCheck, $this->replyToCheck, $this->linkCheck);
 	}
 
 

tests/Unit/Service/Phishing/PhishingDetectionServiceTest.php

@@ -13,6 +13,7 @@
 use OCA\Mail\Service\PhishingDetection\ContactCheck;
 use OCA\Mail\Service\PhishingDetection\CustomEmailCheck;
 use OCA\Mail\Service\PhishingDetection\DateCheck;
+use OCA\Mail\Service\PhishingDetection\LinkCheck;
 use OCA\Mail\Service\PhishingDetection\PhishingDetectionService;
 use OCA\Mail\Service\PhishingDetection\ReplyToCheck;
 
@@ -24,6 +25,7 @@ class PhishingDetectionServiceTest extends TestCase {
 	private CustomEmailCheck|MockObject $customEmailCheck;
 	private DateCheck|MockObject $dateCheck;
 	private ReplyToCheck|MockObject $replyToCheck;
+	private LinkCheck|MockObject $linkCheck;
 	private PhishingDetectionService $service;
 
 	protected function setUp(): void {
@@ -32,7 +34,8 @@ protected function setUp(): void {
 		$this->customEmailCheck = $this->createMock(customEmailCheck::class);
 		$this->dateCheck = $this->createMock(DateCheck::class);
 		$this->replyToCheck = $this->createMock(ReplyToCheck::class);
-		$this->service = new PhishingDetectionService($this->contactCheck, $this->customEmailCheck, $this->dateCheck, $this->replyToCheck);
+		$this->linkCheck = $this->createMock(LinkCheck::class);
+		$this->service = new PhishingDetectionService($this->contactCheck, $this->customEmailCheck, $this->dateCheck, $this->replyToCheck, $this->linkCheck);
 	}
 
 
@@ -56,7 +59,10 @@ public function testCheckHeadersForPhishing(): void {
 		$this->customEmailCheck->expects($this->once())
 			->method('run')
 			->willReturn(new PhishingDetectionResult(PhishingDetectionResult::CUSTOM_EMAIL_CHECK, false));
-		$result = $this->service->checkHeadersForPhishing($parsedHeaders, false);
+		$this->linkCheck->expects($this->once())
+			->method('run')
+			->willReturn(new PhishingDetectionResult(PhishingDetectionResult::LINK_CHECK, false));
+		$result = $this->service->checkHeadersForPhishing($parsedHeaders, true, '');
 		$this->assertFalse($result["warning"]);
 	}