From 2b8e9b43f3bbd05874cf89e615605e0ae6199b27 Mon Sep 17 00:00:00 2001 From: terrtia Date: Fri, 24 Nov 2023 15:05:19 +0100 Subject: [PATCH] chg: [chats] factorise heatmap + chat icon --- bin/importer/FeederImporter.py | 27 +- bin/importer/feeders/BgpMonitor.py | 1 + bin/importer/feeders/Default.py | 2 +- bin/importer/feeders/Discord.py | 38 +++ bin/importer/feeders/Jabber.py | 2 +- bin/importer/feeders/Twitter.py | 2 +- bin/importer/feeders/Urlextract.py | 2 + bin/importer/feeders/abstract_chats_feeder.py | 13 +- bin/lib/chats_viewer.py | 6 +- bin/lib/objects/Chats.py | 4 +- bin/lib/objects/abstract_chat_object.py | 12 +- bin/modules/Exif.py | 2 + var/www/blueprints/objects_image.py | 2 +- .../chats_explorer/block_message.html | 4 +- .../chats_explorer/chat_instance.html | 3 +- .../templates/chats_explorer/chat_viewer.html | 231 +----------------- 16 files changed, 104 insertions(+), 247 deletions(-) create mode 100755 bin/importer/feeders/Discord.py diff --git a/bin/importer/FeederImporter.py b/bin/importer/FeederImporter.py index 4d92cfc6..881cb8a9 100755 --- a/bin/importer/FeederImporter.py +++ b/bin/importer/FeederImporter.py @@ -89,17 +89,26 @@ def importer(self, json_data): feeder_name = feeder.get_name() print(f'importing: {feeder_name} feeder') - obj = feeder.get_obj() # TODO replace by a list of objects to import ???? + # Get Data object: + data_obj = feeder.get_obj() + # process meta if feeder.get_json_meta(): - feeder.process_meta() - - if obj.type == 'item': # object save on disk as file (Items) - gzip64_content = feeder.get_gzip64_content() - return obj, f'{feeder_name} {gzip64_content}' - else: # Messages save on DB - if obj.exists(): - return obj, f'{feeder_name}' + objs = feeder.process_meta() + if objs is None: + objs = set() + else: + objs = set() + + objs.add(data_obj) + + for obj in objs: + if obj.type == 'item': # object save on disk as file (Items) + gzip64_content = feeder.get_gzip64_content() + return obj, f'{feeder_name} {gzip64_content}' + else: # Messages save on DB + if obj.exists(): + return obj, f'{feeder_name}' class FeederModuleImporter(AbstractModule): diff --git a/bin/importer/feeders/BgpMonitor.py b/bin/importer/feeders/BgpMonitor.py index dc926bfd..90c6d7cf 100755 --- a/bin/importer/feeders/BgpMonitor.py +++ b/bin/importer/feeders/BgpMonitor.py @@ -33,3 +33,4 @@ def process_meta(self): tag = 'infoleak:automatic-detection=bgp_monitor' item = Item(self.get_item_id()) item.add_tag(tag) + return set() diff --git a/bin/importer/feeders/Default.py b/bin/importer/feeders/Default.py index ced7ac43..25b3d13b 100755 --- a/bin/importer/feeders/Default.py +++ b/bin/importer/feeders/Default.py @@ -84,4 +84,4 @@ def process_meta(self): Process JSON meta filed. """ # meta = self.get_json_meta() - pass + return set() diff --git a/bin/importer/feeders/Discord.py b/bin/importer/feeders/Discord.py new file mode 100755 index 00000000..1fa02187 --- /dev/null +++ b/bin/importer/feeders/Discord.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Telegram Feeder Importer Module +================ + +Process Telegram JSON + +""" +import os +import sys +import datetime + +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from importer.feeders.abstract_chats_feeder import AbstractChatFeeder +from lib.ConfigLoader import ConfigLoader +from lib.objects import ail_objects +from lib.objects.Chats import Chat +from lib.objects import Messages +from lib.objects import UsersAccount +from lib.objects.Usernames import Username + +import base64 + +class DiscordFeeder(AbstractChatFeeder): + + def __init__(self, json_data): + super().__init__('discord', json_data) + + # def get_obj(self):. + # obj_id = Messages.create_obj_id('telegram', chat_id, message_id, timestamp) + # obj_id = f'message:telegram:{obj_id}' + # self.obj = ail_objects.get_obj_from_global_id(obj_id) + # return self.obj + diff --git a/bin/importer/feeders/Jabber.py b/bin/importer/feeders/Jabber.py index 8c90adfd..1b22bc79 100755 --- a/bin/importer/feeders/Jabber.py +++ b/bin/importer/feeders/Jabber.py @@ -52,4 +52,4 @@ def process_meta(self): # TODO replace me by message user_fr = Username(fr, 'jabber') user_to.add(date, item) user_fr.add(date, item) - return None + return set() diff --git a/bin/importer/feeders/Twitter.py b/bin/importer/feeders/Twitter.py index 1c719e73..3f544e67 100755 --- a/bin/importer/feeders/Twitter.py +++ b/bin/importer/feeders/Twitter.py @@ -45,4 +45,4 @@ def process_meta(self): user = str(self.json_data['meta']['twitter:id']) username = Username(user, 'twitter') username.add(date, item) - return None + return set() diff --git a/bin/importer/feeders/Urlextract.py b/bin/importer/feeders/Urlextract.py index 1ef19920..00080daf 100755 --- a/bin/importer/feeders/Urlextract.py +++ b/bin/importer/feeders/Urlextract.py @@ -56,3 +56,5 @@ def process_meta(self): item = Item(self.item_id) item.set_parent(parent_id) + return set() + diff --git a/bin/importer/feeders/abstract_chats_feeder.py b/bin/importer/feeders/abstract_chats_feeder.py index c075a312..04bdd1e4 100755 --- a/bin/importer/feeders/abstract_chats_feeder.py +++ b/bin/importer/feeders/abstract_chats_feeder.py @@ -131,7 +131,7 @@ def get_obj(self): # TODO handle others objects -> images, pdf, ... self.obj = Messages.Message(obj_id) return self.obj - def process_chat(self, obj, date, timestamp, reply_id=None): # TODO threads + def process_chat(self, new_objs, obj, date, timestamp, reply_id=None): # TODO threads meta = self.json_data['meta']['chat'] # todo replace me by function chat = Chat(self.get_chat_id(), self.get_chat_instance_uuid()) @@ -147,6 +147,12 @@ def process_chat(self, obj, date, timestamp, reply_id=None): # TODO threads if meta.get('date'): # TODO check if already exists chat.set_created_at(int(meta['date']['timestamp'])) + if meta.get('icon'): + img = Images.create(meta['icon'], b64=True) + img.add(date, chat) + chat.set_icon(img.get_global_id()) + new_objs.add(img) + if meta.get('username'): username = Username(meta['username'], self.get_chat_protocol()) chat.update_username_timeline(username.get_global_id(), timestamp) @@ -228,6 +234,7 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS objs = set() if self.obj: objs.add(self.obj) + new_objs = set() date, timestamp = self.get_message_date_timestamp() @@ -261,7 +268,7 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS for obj in objs: # TODO PERF avoid parsing metas multiple times # CHAT - chat = self.process_chat(obj, date, timestamp, reply_id=reply_id) + chat = self.process_chat(new_objs, obj, date, timestamp, reply_id=reply_id) # SENDER # TODO HANDLE NULL SENDER user_account = self.process_sender(obj, date, timestamp) @@ -279,6 +286,8 @@ def process_meta(self): # TODO CHECK MANDATORY FIELDS # -> subchannel ? # -> thread id ? + return new_objs | objs + diff --git a/bin/lib/chats_viewer.py b/bin/lib/chats_viewer.py index 8c201c4b..76af9f6e 100755 --- a/bin/lib/chats_viewer.py +++ b/bin/lib/chats_viewer.py @@ -175,7 +175,7 @@ def get_meta(self, options=set()): if 'chats' in options: meta['chats'] = [] for chat_id in self.get_chats(): - meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'nb_subchannels'})) + meta['chats'].append(Chats.Chat(chat_id, self.uuid).get_meta({'created_at', 'icon', 'nb_subchannels'})) return meta def get_nb_chats(self): @@ -304,7 +304,7 @@ def api_get_chat(chat_id, chat_instance_uuid): chat = Chats.Chat(chat_id, chat_instance_uuid) if not chat.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = chat.get_meta({'created_at', 'img', 'info', 'subchannels', 'username'}) + meta = chat.get_meta({'created_at', 'icon', 'info', 'subchannels', 'username'}) if meta['username']: meta['username'] = get_username_meta_from_global_id(meta['username']) if meta['subchannels']: @@ -325,7 +325,7 @@ def api_get_subchannel(chat_id, chat_instance_uuid): subchannel = ChatSubChannels.ChatSubChannel(chat_id, chat_instance_uuid) if not subchannel.exists(): return {"status": "error", "reason": "Unknown chat"}, 404 - meta = subchannel.get_meta({'chat', 'created_at', 'img', 'nb_messages'}) + meta = subchannel.get_meta({'chat', 'created_at', 'icon', 'nb_messages'}) if meta['chat']: meta['chat'] = get_chat_meta_from_global_id(meta['chat']) if meta.get('username'): diff --git a/bin/lib/objects/Chats.py b/bin/lib/objects/Chats.py index 8c95488d..22a299c5 100755 --- a/bin/lib/objects/Chats.py +++ b/bin/lib/objects/Chats.py @@ -74,8 +74,8 @@ def get_meta(self, options=set()): meta = self._get_meta(options=options) meta['name'] = self.get_name() meta['tags'] = self.get_tags(r_list=True) - if 'img': - meta['icon'] = self.get_img() + if 'icon': + meta['icon'] = self.get_icon() if 'info': meta['info'] = self.get_info() if 'username' in options: diff --git a/bin/lib/objects/abstract_chat_object.py b/bin/lib/objects/abstract_chat_object.py index d266eb5c..6094fb60 100755 --- a/bin/lib/objects/abstract_chat_object.py +++ b/bin/lib/objects/abstract_chat_object.py @@ -113,11 +113,13 @@ def get_name(self): def set_name(self, name): self._set_field('name', name) - def get_img(self): - return self._get_field('img') + def get_icon(self): + icon = self._get_field('icon') + if icon: + return icon.rsplit(':', 1)[1] - def set_img(self, icon): - self._set_field('img', icon) + def set_icon(self, icon): + self._set_field('icon', icon) def get_info(self): return self._get_field('info') @@ -187,7 +189,7 @@ def get_messages(self, start=0, page=1, nb=500, unread=False): # threads ???? # tags = {} messages = {} curr_date = None - for message in self._get_messages(nb=10, page=3): + for message in self._get_messages(nb=30, page=1): timestamp = message[1] date_day = datetime.fromtimestamp(timestamp).strftime('%Y/%m/%d') if date_day != curr_date: diff --git a/bin/modules/Exif.py b/bin/modules/Exif.py index 865cc243..190874d9 100755 --- a/bin/modules/Exif.py +++ b/bin/modules/Exif.py @@ -42,6 +42,8 @@ def compute(self, message): img_exif = img.getexif() print(img_exif) if img_exif: + gps = img_exif.get(34853) + print(gps) for key, val in img_exif.items(): if key in ExifTags.TAGS: print(f'{ExifTags.TAGS[key]}:{val}') diff --git a/var/www/blueprints/objects_image.py b/var/www/blueprints/objects_image.py index 8fd320e8..18d2d715 100644 --- a/var/www/blueprints/objects_image.py +++ b/var/www/blueprints/objects_image.py @@ -40,7 +40,7 @@ def image(filename): abort(404) filename = filename.replace('/', '') image = Images.Image(filename) - return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=True) + return send_from_directory(Images.IMAGE_FOLDER, image.get_rel_path(), as_attachment=False, mimetype='image') @objects_image.route("/objects/images", methods=['GET']) diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html index df2c4d30..3a9db11e 100644 --- a/var/www/templates/chats_explorer/block_message.html +++ b/var/www/templates/chats_explorer/block_message.html @@ -58,7 +58,9 @@ {% endif %} {% if message['images'] %} - + {% for message_image in message['images'] %} + + {% endfor %} {% endif %} 
{{ message['content'] }}
{% for tag in message['tags'] %} diff --git a/var/www/templates/chats_explorer/chat_instance.html b/var/www/templates/chats_explorer/chat_instance.html index d70df757..557ade0a 100644 --- a/var/www/templates/chats_explorer/chat_instance.html +++ b/var/www/templates/chats_explorer/chat_instance.html @@ -77,7 +77,8 @@

{{ chat_instance["protocol"] }} :

{% for chat in chat_instance["chats"] %} - {{ chat['id'] }} + {{ chat['id'] }} {{ chat['name'] }} {{ chat['id'] }} diff --git a/var/www/templates/chats_explorer/chat_viewer.html b/var/www/templates/chats_explorer/chat_viewer.html index bf3df43a..9741fdce 100644 --- a/var/www/templates/chats_explorer/chat_viewer.html +++ b/var/www/templates/chats_explorer/chat_viewer.html @@ -17,6 +17,7 @@ +