chg: [chats] add message file-name object + str emoticon reactions

bin/importer/feeders/abstract_chats_feeder.py

@@ -22,6 +22,8 @@
 from lib.objects import ChatSubChannels
 from lib.objects import Images
 from lib.objects import Messages
+from lib.objects import FilesNames
+# from lib.objects import Files
 from lib.objects import UsersAccount
 from lib.objects.Usernames import Username
 from lib import chats_viewer
@@ -83,6 +85,12 @@ def get_thread_id(self):
     def get_message_id(self):
         return self.json_data['meta']['id']
 
+    def get_media_name(self):
+        return self.json_data['meta'].get('media', {}).get('name')
+
+    def get_reactions(self):
+        return self.json_data['meta'].get('reactions', [])
+
     def get_message_timestamp(self):
         return self.json_data['meta']['date']['timestamp']  # TODO CREATE DEFAULT TIMESTAMP
         # if self.json_data['meta'].get('date'):
@@ -223,6 +231,9 @@ def process_sender(self, new_objs, obj, date, timestamp):
             user_account.set_icon(img.get_global_id())
             new_objs.add(img)
 
+        if meta.get('info'):
+            user_account.set_info(meta['info'])
+
         return user_account
 
     # Create abstract class: -> new API endpoint ??? => force field, check if already imported ?
@@ -251,11 +262,22 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
 
         print(self.obj.type)
 
+        # TODO FILES + FILES REF
+
         # get object by meta object type
         if self.obj.type == 'message':
             # Content
             obj = Messages.create(self.obj.id, self.get_message_content())  # TODO translation
 
+            # FILENAME
+            media_name = self.get_media_name()
+            if media_name:
+                print(media_name)
+                FilesNames.FilesNames().create(media_name, date, obj)
+
+            for reaction in self.get_reactions():
+                obj.add_reaction(reaction['reaction'], int(reaction['count']))
+
         else:
             chat_id = self.get_chat_id()
             message_id = self.get_message_id()
@@ -271,6 +293,11 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
                 obj.add(date, message)
                 obj.set_parent(obj_global_id=message.get_global_id())
 
+                # FILENAME
+                media_name = self.get_media_name()
+                if media_name:
+                    FilesNames.FilesNames().create(media_name, date, message, file_obj=obj)
+
         for obj in objs:  # TODO PERF avoid parsing metas multiple times
 
             # CHAT

bin/lib/ail_core.py

@@ -16,7 +16,8 @@
 r_object = config_loader.get_db_conn("Kvrocks_Objects")
 config_loader = None
 
-AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon', 'hhhash',
+AIL_OBJECTS = sorted({'chat', 'cookie-name', 'cve', 'cryptocurrency', 'decoded', 'domain', 'etag', 'favicon',
+                      'file-name', 'hhhash',
                       'item', 'image', 'message', 'pgp', 'screenshot', 'title', 'user-account', 'username'})
 
 def get_ail_uuid():

bin/lib/correlations_engine.py

@@ -49,10 +49,11 @@
     "domain": ["cve", "cookie-name", "cryptocurrency", "decoded", "etag", "favicon", "hhhash", "item", "pgp", "title", "screenshot", "username"],
     "etag": ["domain"],
     "favicon": ["domain", "item"],  # TODO Decoded
+    "file-name": ["chat", "message"],
     "hhhash": ["domain"],
     "image": ["chat", "message", "user-account"],
     "item": ["cve", "cryptocurrency", "decoded", "domain", "favicon", "pgp", "screenshot", "title", "username"],  # chat ???
-    "message": ["cve", "cryptocurrency", "decoded", "image", "pgp", "user-account"],  # chat ??
+    "message": ["cve", "cryptocurrency", "decoded", "file-name", "image", "pgp", "user-account"],  # chat ??
     "pgp": ["domain", "item", "message"],
     "screenshot": ["domain", "item"],
     "title": ["domain", "item"],

bin/lib/objects/FilesNames.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+
+from flask import url_for
+from pymisp import MISPObject
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
+
+config_loader = ConfigLoader()
+r_object = config_loader.get_db_conn("Kvrocks_Objects")
+config_loader = None
+
+
+class FileName(AbstractDaterangeObject):
+    """
+    AIL FileName Object. (strings)
+    """
+
+    # ID = SHA256
+    def __init__(self, name):
+        super().__init__('file-name', name)
+
+    # def get_ail_2_ail_payload(self):
+    #     payload = {'raw': self.get_gzip_content(b64=True),
+    #                 'compress': 'gzip'}
+    #     return payload
+
+    # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
+    def delete(self):
+        # # TODO:
+        pass
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('correlation.show_correlation', type=self.type, id=self.id)
+        else:
+            url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
+        return url
+
+    def get_svg_icon(self):
+        return {'style': 'far', 'icon': '\uf249', 'color': '#36F5D5', 'radius': 5}
+
+    def get_misp_object(self):
+        obj_attrs = []
+        obj = MISPObject('file')
+
+        # obj_attrs.append(obj.add_attribute('sha256', value=self.id))
+        # obj_attrs.append(obj.add_attribute('attachment', value=self.id, data=self.get_file_content()))
+        for obj_attr in obj_attrs:
+            for tag in self.get_tags():
+                obj_attr.add_tag(tag)
+        return obj
+
+    def get_meta(self, options=set()):
+        meta = self._get_meta(options=options)
+        meta['id'] = self.id
+        meta['tags'] = self.get_tags(r_list=True)
+        if 'tags_safe' in options:
+            meta['tags_safe'] = self.is_tags_safe(meta['tags'])
+        return meta
+
+    def create(self): # create ALL SET ??????
+        pass
+
+    def add_reference(self, date, src_ail_object, file_obj=None):
+        self.add(date, src_ail_object)
+        if file_obj:
+            self.add_correlation(file_obj.type, file_obj.get_subtype(r_str=True), file_obj.get_id())
+
+# TODO USE ZSET FOR ALL OBJS IDS ??????
+
+class FilesNames(AbstractDaterangeObjects):
+    """
+        CookieName Objects
+    """
+    def __init__(self):
+        super().__init__('file-name', FileName)
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search
+
+    # TODO sanitize file name
+    def create(self, name, date, src_ail_object, file_obj=None, limit=500, force=False):
+        if 0 < len(name) <= limit or force or limit < 0:
+            file_name = self.obj_class(name)
+            # if not file_name.exists():
+            #     file_name.create()
+            file_name.add_reference(date, src_ail_object, file_obj=file_obj)
+            return file_name
+
+# if __name__ == '__main__':
+#     name_to_search = '29ba'
+#     print(search_screenshots_by_name(name_to_search))

bin/lib/objects/Messages.py

@@ -118,9 +118,24 @@ def get_user_account(self, meta=False):
             user_account = f'user-account:{user_account["user-account"].pop()}'
             if meta:
                 _, user_account_subtype, user_account_id = user_account.split(':', 3)
-                user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'username', 'username_meta'})
+                user_account = UsersAccount.UserAccount(user_account_id, user_account_subtype).get_meta(options={'icon', 'username', 'username_meta'})
         return  user_account
 
+    def get_files_names(self):
+        names = []
+        filenames = self.get_correlation('file-name').get('file-name')
+        if filenames:
+            for name in filenames:
+                names.append(name[1:])
+        return names
+
+    def get_reactions(self):
+        return r_object.hgetall(f'meta:reactions:{self.type}::{self.id}')
+
+    # TODO sanitize reactions
+    def add_reaction(self, reactions, nb_reaction):
+        r_object.hset(f'meta:reactions:{self.type}::{self.id}', reactions, nb_reaction)
+
     # Update value on import
     # reply to -> parent ?
     # reply/comment - > children ?
@@ -232,6 +247,10 @@ def get_meta(self, options=None, timestamp=None):
             meta['chat'] = self.get_chat_id()
         if 'images' in options:
             meta['images'] = self.get_images()
+        if 'files-names' in options:
+            meta['files-names'] = self.get_files_names()
+        if 'reactions' in options:
+            meta['reactions'] = self.get_reactions()
 
         # meta['encoding'] = None
         return meta
@@ -314,8 +333,8 @@ def create_obj_id(chat_instance, chat_id, message_id, timestamp, channel_id=None
 # def create(source, chat_id, message_id, timestamp, content, tags=[]):
 def create(obj_id, content, translation=None, tags=[]):
     message = Message(obj_id)
-    if not message.exists():
-        message.create(content, translation=translation, tags=tags)
+    # if not message.exists():
+    message.create(content, translation=translation, tags=tags)
     return message
 
 

bin/lib/objects/UsersAccount.py

@@ -91,6 +91,12 @@ def get_icon(self):
     def set_icon(self, icon):
         self._set_field('icon', icon)
 
+    def get_info(self):
+        return self._get_field('info')
+
+    def set_info(self, info):
+        return self._set_field('info', info)
+
     def _get_timeline_username(self):
         return Timeline(self.get_global_id(), 'username')
 

bin/lib/objects/abstract_chat_object.py

@@ -181,15 +181,15 @@ def get_nb_message_this_week(self):
 
     def get_message_meta(self, message, timestamp=None):  # TODO handle file message
         message = Messages.Message(message[9:])
-        meta = message.get_meta(options={'content', 'images', 'link', 'parent', 'parent_meta', 'user-account'}, timestamp=timestamp)
+        meta = message.get_meta(options={'content', 'files-names', 'images', 'link', 'parent', 'parent_meta', 'reactions', 'user-account'}, timestamp=timestamp)
         return meta
 
     def get_messages(self, start=0, page=1, nb=500, unread=False):  # threads ???? # TODO ADD last/first message timestamp + return page
         # TODO return message meta
         tags = {}
         messages = {}
         curr_date = None
-        for message in self._get_messages(nb=30, page=1):
+        for message in self._get_messages(nb=50, page=1):
             timestamp = message[1]
             date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
             if date_day != curr_date:

bin/lib/objects/ail_objects.py

@@ -21,6 +21,7 @@
 from lib.objects.Domains import Domain
 from lib.objects import Etags
 from lib.objects.Favicons import Favicon
+from lib.objects import FilesNames
 from lib.objects import HHHashs
 from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
 from lib.objects import Images
@@ -69,6 +70,8 @@ def get_object(obj_type, subtype, obj_id):
         return Etags.Etag(obj_id)
     elif obj_type == 'favicon':
         return Favicon(obj_id)
+    elif obj_type == 'file-name':
+        return FilesNames.FileName(obj_id)
     elif obj_type == 'hhhash':
         return HHHashs.HHHash(obj_id)
     elif obj_type == 'image':

var/www/templates/chats_explorer/block_message.html

@@ -63,7 +63,17 @@
                 <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
             {%  endfor %}
         {% endif %}
+        {% if message['files-names'] %}
+            {% for file_name in message['files-names'] %}
+                <div class="flex-shrink-1 bg-white border-primary text-secondary rounded py-2 px-3 ml-4 mb-3" style="overflow-x: auto">
+                    <i class="far fa-file fa-3x"></i> {{ file_name }}
+                </div>
+            {%  endfor %}
+        {% endif %}
         <pre class="my-0">{{ message['content'] }}</pre>
+        {% for reaction in message['reactions'] %}
+            <span class="border rounded px-1">{{ reaction }} {{ message['reactions'][reaction] }}</span>
+        {%  endfor %}
         {% for tag in message['tags'] %}
             <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
         {%  endfor %}