Merge pull request #807 from vodolaz095/yandex_normalize_dedup

added normalisation for yandex based emails

lib/normalizeEmail.js

@@ -60,6 +60,9 @@ var outlookdotcom_domains = ['hotmail.at', 'hotmail.be', 'hotmail.ca', 'hotmail.
 // This list is likely incomplete
 var yahoo_domains = ['rocketmail.com', 'yahoo.ca', 'yahoo.co.uk', 'yahoo.com', 'yahoo.de', 'yahoo.fr', 'yahoo.in', 'yahoo.it', 'ymail.com'];
 
+// List of domains used by yandex.ru
+var yandex_domains = ['yandex.ru', 'yandex.ua', 'yandex.kz', 'yandex.com', 'yandex.by', 'ya.ru'];
+
 // replace single dots, but not multiple consecutive dots
 function dotsReplacer(match) {
   if (match.length > 1) {
@@ -129,6 +132,11 @@ function normalizeEmail(email, options) {
     if (options.all_lowercase || options.yahoo_lowercase) {
       parts[0] = parts[0].toLowerCase();
     }
+  } else if (~yandex_domains.indexOf(parts[1])) {
+    if (options.all_lowercase || options.yahoo_lowercase) {
+      parts[0] = parts[0].toLowerCase();
+    }
+    parts[1] = 'yandex.ru'; // all yandex domains are equal, 1st preffered
   } else if (options.all_lowercase) {
     // Any other address
     parts[0] = parts[0].toLowerCase();

src/lib/normalizeEmail.js

@@ -145,6 +145,16 @@ const yahoo_domains = [
   'ymail.com',
 ];
 
+// List of domains used by yandex.ru
+const yandex_domains = [
+  'yandex.ru',
+  'yandex.ua',
+  'yandex.kz',
+  'yandex.com',
+  'yandex.by',
+  'ya.ru',
+];
+
 // replace single dots, but not multiple consecutive dots
 function dotsReplacer(match) {
   if (match.length > 1) {
@@ -214,6 +224,11 @@ export default function normalizeEmail(email, options) {
     if (options.all_lowercase || options.yahoo_lowercase) {
       parts[0] = parts[0].toLowerCase();
     }
+  } else if (~yandex_domains.indexOf(parts[1])) {
+    if (options.all_lowercase || options.yahoo_lowercase) {
+      parts[0] = parts[0].toLowerCase();
+    }
+    parts[1] = 'yandex.ru'; // all yandex domains are equal, 1st preffered
   } else if (options.all_lowercase) {
     // Any other address
     parts[0] = parts[0].toLowerCase();

test/sanitizers.js

@@ -238,6 +238,12 @@ describe('Sanitizers', function () {
         '[email protected]': '[email protected]',
         '[email protected]': '[email protected]',
         '"foo@bar"@baz.com': '"foo@bar"@baz.com',
+        '[email protected]': '[email protected]',
+        '[email protected]': '[email protected]',
+        '[email protected]': '[email protected]',
+        '[email protected]': '[email protected]',
+        '[email protected]': '[email protected]',
+        '[email protected]': '[email protected]',
       },
     });
 

validator.js

@@ -1456,6 +1456,9 @@ var outlookdotcom_domains = ['hotmail.at', 'hotmail.be', 'hotmail.ca', 'hotmail.
 // This list is likely incomplete
 var yahoo_domains = ['rocketmail.com', 'yahoo.ca', 'yahoo.co.uk', 'yahoo.com', 'yahoo.de', 'yahoo.fr', 'yahoo.in', 'yahoo.it', 'ymail.com'];
 
+// List of domains used by yandex.ru
+var yandex_domains = ['yandex.ru', 'yandex.ua', 'yandex.kz', 'yandex.com', 'yandex.by', 'ya.ru'];
+
 // replace single dots, but not multiple consecutive dots
 function dotsReplacer(match) {
   if (match.length > 1) {
@@ -1525,6 +1528,11 @@ function normalizeEmail(email, options) {
     if (options.all_lowercase || options.yahoo_lowercase) {
       parts[0] = parts[0].toLowerCase();
     }
+  } else if (~yandex_domains.indexOf(parts[1])) {
+    if (options.all_lowercase || options.yahoo_lowercase) {
+      parts[0] = parts[0].toLowerCase();
+    }
+    parts[1] = 'yandex.ru'; // all yandex domains are equal, 1st preffered
   } else if (options.all_lowercase) {
     // Any other address
     parts[0] = parts[0].toLowerCase();