feat(unicode): strip additional rune ranges (#1575)

pelias · Nov 11, 2021 · 35efea5 · 35efea5
1 parent 9021f09
commit 35efea5
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/helper/unicode.js b/helper/unicode.js
@@ -73,10 +73,12 @@ const MISC_UNSUPPORTED_SYMBOLS = regenerate()
   .addRange(0x1D400, 0x1D7FF) // Mathematical Alphanumeric Symbols (1D400-1D7FF)
 
   // emojis
+  .addRange(0x1F1E6, 0x1F1FF) // Regional indicator symbol (1F1E6-1F1FF)
   .addRange(0x1F300, 0x1F5FF) // Miscellaneous Symbols and Pictographs (1F300-1F5FF)
   .addRange(0x1F3FB, 0x1F3FF) // Emoji Modifier Fitzpatrick (skin tones) (1F3FB–1F3FF)
   .addRange(0x1F600, 0x1F64F) // Emoticons (1F600–1F64F)
   .addRange(0x1F680, 0x1F6FF) // Transport and Map Symbols (1F680-1F6FF)
+  .addRange(0x1F780, 0x1F7FF) // Geometric Shapes Extended (1F780-1F7FF)
   .addRange(0x1F900, 0x1F9FF) // Supplemental Symbols and Pictographs (1F900-1F9FF)
   .toRegExp('g');
 

diff --git a/test/unit/helper/unicode.js b/test/unit/helper/unicode.js
@@ -47,6 +47,13 @@ module.exports.tests.normalize = function (test) {
     t.equal(norm('👩‍❤️‍👩'), '', 'complex emoji ZWJ sequence (6 codepoints)');
     t.end();
   });
+
+  // https://github.com/pelias/api/issues/1574
+  test('normalize: issue 1574', function (t) {
+    const input = 'ASDFJK СТРАНЫ БЫЛИ КТО СЛУШАЕТ МЕНЯФ1💒🌋🚭🚱⬆️↗️➡️↘️⬇️↙️⬅️↖️↕️↔️↩↪⤴️👯‍♂️🇰🇿⤵️🔃🔄🛐⚛🕉✡☸♍☯️☸✡🕉🛐⚛🔅🔆🔴🟠🟡🟢НЗЕВИ Л К ВО УНТВА854У32Й1🔵🟣🟤⚪⚫⚫🟥🟧🟨🔶️🟩🟦🟪🟫⬛⬜◼◻◾◽💠'; // jshint ignore:line
+    t.equal(norm(input), 'ASDFJK СТРАНЫ БЫЛИ КТО СЛУШАЕТ МЕНЯФ1НЗЕВИ Л К ВО УНТВА854У32Й1', 'issue 1574');
+    t.end();
+  });
 };
 
 module.exports.all = function (tape, common) {