chg: [chats] factorise heatmap + chat icon

bin/importer/FeederImporter.py

@@ -89,17 +89,26 @@ def importer(self, json_data):
         feeder_name = feeder.get_name()
         print(f'importing: {feeder_name} feeder')
 
-        obj = feeder.get_obj()  # TODO replace by a list of objects to import ????
+        # Get Data object:
+        data_obj = feeder.get_obj()
+
         # process meta
         if feeder.get_json_meta():
-            feeder.process_meta()
-
-        if obj.type == 'item':  # object save on disk as file (Items)
-            gzip64_content = feeder.get_gzip64_content()
-            return obj, f'{feeder_name} {gzip64_content}'
-        else:  # Messages save on DB
-            if obj.exists():
-                return obj, f'{feeder_name}'
+            objs = feeder.process_meta()
+            if objs is None:
+                objs = set()
+        else:
+            objs = set()
+
+        objs.add(data_obj)
+
+        for obj in objs:
+            if obj.type == 'item':  # object save on disk as file (Items)
+                gzip64_content = feeder.get_gzip64_content()
+                return obj, f'{feeder_name} {gzip64_content}'
+            else:  # Messages save on DB
+                if obj.exists():
+                    return obj, f'{feeder_name}'
 
 
 class FeederModuleImporter(AbstractModule):

bin/importer/feeders/BgpMonitor.py

@@ -33,3 +33,4 @@ def process_meta(self):
         tag = 'infoleak:automatic-detection=bgp_monitor'
         item = Item(self.get_item_id())
         item.add_tag(tag)
+        return set()

bin/importer/feeders/Default.py

@@ -84,4 +84,4 @@ def process_meta(self):
         Process JSON meta filed.
         """
         # meta = self.get_json_meta()
-        pass
+        return set()

bin/importer/feeders/Discord.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The Telegram Feeder Importer Module
+================
+
+Process Telegram JSON
+
+"""
+import os
+import sys
+import datetime
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from importer.feeders.abstract_chats_feeder import AbstractChatFeeder
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import ail_objects
+from lib.objects.Chats import Chat
+from lib.objects import Messages
+from lib.objects import UsersAccount
+from lib.objects.Usernames import Username
+
+import base64
+
+class DiscordFeeder(AbstractChatFeeder):
+
+    def __init__(self, json_data):
+        super().__init__('discord', json_data)
+
+    # def get_obj(self):.
+    #     obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp)
+    #     obj_id = f'message:telegram:{obj_id}'
+    #     self.obj = ail_objects.get_obj_from_global_id(obj_id)
+    #     return self.obj
+

bin/importer/feeders/Jabber.py

@@ -52,4 +52,4 @@ def process_meta(self): # TODO replace me by message
         user_fr = Username(fr, 'jabber')
         user_to.add(date, item)
         user_fr.add(date, item)
-        return None
+        return set()

bin/importer/feeders/Twitter.py

@@ -45,4 +45,4 @@ def process_meta(self):
         user = str(self.json_data['meta']['twitter:id'])
         username = Username(user, 'twitter')
         username.add(date, item)
-        return None
+        return set()

bin/importer/feeders/Urlextract.py

@@ -56,3 +56,5 @@ def process_meta(self):
             item = Item(self.item_id)
             item.set_parent(parent_id)
 
+        return set()
+

bin/importer/feeders/abstract_chats_feeder.py

@@ -131,7 +131,7 @@ def get_obj(self):  # TODO handle others objects -> images, pdf, ...
             self.obj = Messages.Message(obj_id)
         return self.obj
 
-    def process_chat(self, obj, date, timestamp, reply_id=None):  # TODO threads
+    def process_chat(self, new_objs, obj, date, timestamp, reply_id=None):  # TODO threads
         meta = self.json_data['meta']['chat'] # todo replace me by function
         chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid())
 
@@ -147,6 +147,12 @@ def process_chat(self, obj, date, timestamp, reply_id=None):  # TODO threads
         if meta.get('date'): # TODO check if already exists
             chat.set_created_at(int(meta['date']['timestamp']))
 
+        if meta.get('icon'):
+            img = Images.create(meta['icon'], b64=True)
+            img.add(date, chat)
+            chat.set_icon(img.get_global_id())
+            new_objs.add(img)
+
         if meta.get('username'):
             username = Username(meta['username'], self.get_chat_protocol())
             chat.update_username_timeline(username.get_global_id(), timestamp)
@@ -228,6 +234,7 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
         objs = set()
         if self.obj:
             objs.add(self.obj)
+        new_objs = set()
 
         date, timestamp = self.get_message_date_timestamp()
 
@@ -261,7 +268,7 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
         for obj in objs:  # TODO PERF avoid parsing metas multiple times
 
             # CHAT
-            chat = self.process_chat(obj, date, timestamp, reply_id=reply_id)
+            chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id)
 
             # SENDER # TODO HANDLE NULL SENDER
             user_account = self.process_sender(obj, date, timestamp)
@@ -279,6 +286,8 @@ def process_meta(self):  # TODO CHECK MANDATORY FIELDS
                 #       -> subchannel ?
                 #       -> thread id ?
 
+        return new_objs | objs
+
 
 
 

bin/lib/chats_viewer.py

@@ -175,7 +175,7 @@ def get_meta(self, options=set()):
         if 'chats' in options:
             meta['chats'] = []
             for chat_id in self.get_chats():
-                meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'}))
+                meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'}))
         return meta
 
     def get_nb_chats(self):
@@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid):
     chat = Chats.Chat(chat_id, chat_instance_uuid)
     if not chat.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
-    meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'})
+    meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'})
     if meta['username']:
         meta['username'] = get_username_meta_from_global_id(meta['username'])
     if meta['subchannels']:
@@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid):
     subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid)
     if not subchannel.exists():
         return {"status": "error", "reason": "Unknown chat"}, 404
-    meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'})
+    meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'})
     if meta['chat']:
         meta['chat'] = get_chat_meta_from_global_id(meta['chat'])
     if meta.get('username'):

bin/lib/objects/Chats.py

@@ -74,8 +74,8 @@ def get_meta(self, options=set()):
         meta = self._get_meta(options=options)
         meta['name'] = self.get_name()
         meta['tags'] = self.get_tags(r_list=True)
-        if 'img':
-            meta['icon'] = self.get_img()
+        if 'icon':
+            meta['icon'] = self.get_icon()
         if 'info':
             meta['info'] = self.get_info()
         if 'username' in options:

bin/lib/objects/abstract_chat_object.py

@@ -113,11 +113,13 @@ def get_name(self):
     def set_name(self, name):
         self._set_field('name', name)
 
-    def get_img(self):
-        return self._get_field('img')
+    def get_icon(self):
+        icon = self._get_field('icon')
+        if icon:
+            return icon.rsplit(':', 1)[1]
 
-    def set_img(self, icon):
-        self._set_field('img', icon)
+    def set_icon(self, icon):
+        self._set_field('icon', icon)
 
     def get_info(self):
         return self._get_field('info')
@@ -187,7 +189,7 @@ def get_messages(self, start=0, page=1, nb=500, unread=False):  # threads ???? #
         tags = {}
         messages = {}
         curr_date = None
-        for message in self._get_messages(nb=10, page=3):
+        for message in self._get_messages(nb=30, page=1):
             timestamp = message[1]
             date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d')
             if date_day != curr_date:

bin/modules/Exif.py

@@ -42,6 +42,8 @@ def compute(self, message):
         img_exif = img.getexif()
         print(img_exif)
         if img_exif:
+            gps = img_exif.get(34853)
+            print(gps)
             for key, val in img_exif.items():
                 if key in ExifTags.TAGS:
                     print(f'{ExifTags.TAGS[key]}:{val}')

var/www/blueprints/objects_image.py

@@ -40,7 +40,7 @@ def image(filename):
         abort(404)
     filename = filename.replace('/', '')
     image = Images.Image(filename)
-    return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True)
+    return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image')
 
 
 @objects_image.route("/objects/images", methods=['GET'])

var/www/templates/chats_explorer/block_message.html

@@ -58,7 +58,9 @@
             </div>
         {% endif %}
         {% if message['images'] %}
-            <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message['images'][0])}}">
+            {% for message_image in message['images'] %}
+                <img class="message_image mb-1" src="{{ url_for('objects_image.image', filename=message_image)}}">
+            {%  endfor %}
         {% endif %}
         <pre class="my-0">{{ message['content'] }}</pre>
         {% for tag in message['tags'] %}

var/www/templates/chats_explorer/chat_instance.html

@@ -77,7 +77,8 @@ <h4 class="text-secondary">{{ chat_instance["protocol"] }} :</h4>
                     {% for chat in chat_instance["chats"] %}
                         <tr>
                             <td>
-                                <img src="{{ url_for('static', filename='image/ail-icon.png') }}" class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
+                                <img src="{% if chat['icon'] %}{{ url_for('objects_image.image', filename=chat['icon'])}}{% else %}{{ url_for('static', filename='image/ail-icon.png') }}{% endif %}"
+                                     class="rounded-circle mr-1" alt="{{ chat['id'] }}" width="40" height="40">
                             </td>
                             <td><b>{{ chat['name'] }}</b></td>
                             <td><a href="{{ url_for('chats_explorer.chats_explorer_chat') }}?uuid={{ chat_instance['uuid'] }}&id={{ chat['id'] }}">{{ chat['id'] }}</a></td>