#14 initial emoji keywords

package.json

@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "cldr-annotations-derived-full": "latest"
+    "cldr-annotations-full": "latest"
   }
 }

requirements.txt

@@ -2,6 +2,7 @@ beautifulsoup4==4.9.3
 black>=19.10b0
 certifi>=2020.12.5
 defusedxml==0.7.1
+emoji>=2.2.0
 mwparserfromhell==0.6
 packaging>=20.9
 pytest-cov>=3.0.0

src/scribe_data/extract_transform/process_unicode.py

@@ -8,6 +8,7 @@
     gen_emoji_autosuggestions
 """
 
+import emoji
 import json
 
 from scribe_data.load.update_utils import get_language_iso
@@ -51,14 +52,26 @@ def gen_emoji_autosuggestions(
 
     iso = get_language_iso(language)
 
-    cldr_file_path = f"node_modules/cldr-annotations-derived-full/annotationsDerived/{iso}/annotations.json"
+    cldr_file_path = f"node_modules/cldr-annotations-full/annotations/{iso}/annotations.json"
 
     with open(cldr_file_path, "r") as file:
         cldr_data = json.load(file)
 
-    emoji_dict = cldr_data["annotationsDerived"]["annotations"]
+    cldr_dict = cldr_data["annotations"]["annotations"]
 
-    print("Number of emojis loaded:", len(emoji_dict))
+    print("Number of characters loaded:", len(cldr_dict))
+
+    for cldr_char in cldr_dict:
+        # Filter CLDR data for emoji characters
+        if cldr_char in emoji.EMOJI_DATA:
+            emoji_annotations = cldr_dict[cldr_char]
+
+            for emoji_keyword in emoji_annotations["default"]:
+                # Use single-word annotations as keywords
+                if len(emoji_keyword.split()) == 1:
+                    autosuggest_dict.setdefault(emoji_keyword, []).append(cldr_char)
+
+    print("Number of trigger keywords found:", len(autosuggest_dict))
 
     ###