Merge pull request #2 from matesh/test

V1.1.0

README.md

@@ -22,8 +22,8 @@
     + [Linux - Running MQTTk](#linux---running-mqttk)
 - [Using the app](#using-the-app)
   * [Features](#features)
+  * [Screenshots](#screenshots)
   * [Planned features](#planned-features)
-    + [V1.1](#v11)
     + [V1.2](#v12)
     + [V1.3](#v13)
 - [Building the app from source](#building-the-app-from-source)
@@ -37,6 +37,7 @@
   * [macOS universal2 appimage](#macos-universal2-appimage)
   * [Linux binary package or app](#linux-binary-package-or-app)
 
+
 # Introduction
 MQTTk is a very lightweight MQTT GUI client that looks retarded, but it does the job fast in a native
 fashion, without bloated and sluggish browser, java and javascript based rubbish that may look good, but
@@ -203,30 +204,39 @@ pretty formatter and hex decoder to analyse message data.
 You can also publish messages, save message templates and one-click publish them. You can send messages with
 any QoS and retained messages as well.
 
-So far, I added the most useful message decoding features: JSON pretty formatter and hex decoders. 
+So far, I added the most useful message decoding features: JSON pretty formatter and hex decoders. There is also an
+option to attempt to decompress the payload before feeding it into the decoder.
 
 If you used MQTT.fx in the past, MQTTk will try to find and import your config. The connection profiles,
 subscribe and publish history will be imported, as well as the saved message templates.
 
 There is a built-in log feature to show any exceptions/debug information, let me know if you see something
 unusual there.
 
+Connection profiles, subscription and publish history and saved message templates can be exported and imported.
+
+Messages can be dumped into .CSV and .JSON formats. Message payload is exported as unicode text if possible,
+otherwise it is encoded in base64.
+
+## Screenshots
+
 Subscribe interface
+
 ![Subscribe interface](/assets/subscribe.png)
 
 Publish interface
+
 ![Publish interface](/assets/publish.png)
 
 Configuration interface
+
 ![Configuration interface](/assets/configuration.png)
 
-## Planned features
-### V1.1
-- Export and import MQTTk configuration
-- Export and import subscribe topic history, publish topic history and templates
-- Message dump
-- Log output to file and better logging
+Export subscribe/publish history interface
 
+![Configuration interface](/assets/export.png)
+
+## Planned features
 ### V1.2
 - tree-style topic inspector where all incoming messages are organised in a tree and the latest payload is shown
 

mqttk.spec

@@ -49,4 +49,4 @@ app = BUNDLE(exe,
              name='mqttk.app',
              icon='mqttk.icns',
              bundle_identifier='com.mateszabo.mqttk',
-             version='0.1.0')
+             version='1.1.0')

mqttk/__init__.py

@@ -1 +1 @@
-__version__ = '1.0.0'
+__version__ = '1.1.0'

mqttk/__main__.py

@@ -16,12 +16,15 @@
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <https://www.gnu.org/licenses/>.
 """
-
+import json
 import os
 import traceback
 from datetime import datetime
 import sys
 import time
+from pathlib import Path
+from functools import partial
+import base64
 try:
     import tkinter as tk
     import tkinter.ttk as ttk
@@ -42,7 +45,7 @@
 from mqttk.widgets.publish_tab import PublishTab
 from mqttk.constants import CONNECT, DISCONNECT
 from mqttk.widgets.log_tab import LogTab
-from mqttk.widgets.dialogs import AboutDialog, SplashScreen
+from mqttk.widgets.dialogs import AboutDialog, SplashScreen, ConnectionConfigImportExport, SubscribePublishImportExport
 from mqttk.widgets.configuration_dialog import ConfigurationWindow
 from mqttk.config_handler import ConfigHandler
 from mqttk.MQTT_manager import MqttManager
@@ -64,9 +67,10 @@ class PotatoLog:
     def __init__(self):
         self.add_message_callback = None
         self.message_queue = []
+        self.config_handler = None
 
     def add_message(self, message_level, *args):
-        message = "{} - {} ".format(datetime.now().strftime("%H:%M:%S.%f"), message_level)
+        message = "{} - {} ".format(datetime.now().strftime("%Y/%m/%d, %H:%M:%S.%f"), message_level)
         message += ", ".join([str(x) for x in args])
         message += os.linesep
         if self.add_message_callback is None:
@@ -75,8 +79,10 @@ def add_message(self, message_level, *args):
             if len(self.message_queue) != 0:
                 for queued_message in self.message_queue:
                     self.add_message_callback(queued_message)
+                    self.config_handler.add_log_message(queued_message)
                 self.message_queue = []
             self.add_message_callback(message)
+            self.config_handler.add_log_message(message)
 
     def warning(self, *args):
         message_level = "[W]"
@@ -109,13 +115,12 @@ class App:
     def __init__(self, root):
         self.log = PotatoLog()
         self.config_handler = ConfigHandler(self.log)
+        self.log.config_handler = self.config_handler
 
         self.mqtt_manager = None
         self.current_connection_configuration = None
 
         root.title("MQTTk")
-        # root.createcommand('tk::mac::ShowPreferences', self.show_preferences)  # set preferences menu
-        root.createcommand('tk::mac::ShowHelp', self.on_about_menu)
 
         # Restore window size and position, if not available or out of bounds, reset to default
         screenwidth = root.winfo_screenwidth()
@@ -156,6 +161,10 @@ def __init__(self, root):
             self.style.theme_use('winnative')
         if sys.platform == "darwin":
             self.style.theme_use("default")  # aqua, clam, alt, default, classic
+            root.createcommand('tk::mac::ReopenApplication', root.deiconify)
+            root.createcommand('tk::mac::ShowHelp', self.on_about_menu)
+            # root.createcommand('tk::mac::ShowPreferences', self.show_preferences)  # set preferences menu
+
         self.style.configure("New.TFrame", background="#b3ffb5")
         self.style.configure("New.TLabel", background="#b3ffb5")
         self.style.configure("Selected.TFrame", background="#96bfff")
@@ -166,18 +175,39 @@ def __init__(self, root):
         self.menubar = tk.Menu(root, background=self.style.lookup("TLabel", "background"),
                                foreground=self.style.lookup("TLabel", "foreground"))
         self.root.config(menu=self.menubar)
+
         self.file_menu = tk.Menu(self.menubar, background=self.style.lookup("TLabel", "background"),
                                  foreground=self.style.lookup("TLabel", "foreground"))
 
         self.import_menu = tk.Menu(self.menubar, background=self.style.lookup("TLabel", "background"),
                                    foreground=self.style.lookup("TLabel", "foreground"))
-        self.about_menu = tk.Menu(self.menubar)
+
+        self.about_menu = tk.Menu(self.menubar, background=self.style.lookup("TLabel", "background"),
+                                  foreground=self.style.lookup("TLabel", "foreground"))
+
+        self.export_menu = tk.Menu(self.menubar, background=self.style.lookup("TLabel", "background"),
+                                   foreground=self.style.lookup("TLabel", "foreground"))
+
+        self.export_messages_menu = tk.Menu(self.menubar, background=self.style.lookup("TLabel", "background"),
+                                            foreground=self.style.lookup("TLabel", "foreground"))
+
         self.menubar.add_cascade(menu=self.file_menu, label="File")
+        self.file_menu.add_command(label="Exit", command=self.on_exit)
+
         self.menubar.add_cascade(menu=self.import_menu, label="Import")
+        self.import_menu.add_command(label="MQTT.fx config", command=self.import_mqttfx_config)
+        self.import_menu.add_command(label="Connection configuration", command=self.import_connection_config)
+        self.import_menu.add_command(label="Subscribe/publish content", command=self.import_subscribe_publish)
+
+        self.menubar.add_cascade(menu=self.export_menu, label="Export")
+        self.export_menu.add_cascade(menu=self.export_messages_menu, label="Messages")
+        self.export_messages_menu.add_command(label="JSON", command=partial(self.export_messages, format="JSON"))
+        self.export_messages_menu.add_command(label="CSV", command=partial(self.export_messages, format="CSV"))
+        self.export_menu.add_command(label="Connection configuration", command=self.export_connection_config)
+        self.export_menu.add_command(label="Subscribe/publish content", command=self.export_subscribe_publish)
+
         self.menubar.add_cascade(menu=self.about_menu, label="Help")
-        self.file_menu.add_command(label="Exit", command=self.on_exit)
         self.about_menu.add_command(label="About MQTTk", command=self.on_about_menu)
-        self.import_menu.add_command(label="Import MQTT.fx config", command=self.import_mqttfx_config)
 
         self.main_window_frame = ttk.Frame(root)
         self.main_window_frame.pack(fill='both', expand=1)
@@ -289,6 +319,71 @@ def import_mqttfx_config(self):
         if success:
             self.on_config_update()
 
+    def export_messages(self, format):
+        if len(self.subscribe_frame.messages) == 0:
+            messagebox.showinfo("Info", "The message list is empty")
+            return
+
+        output_location = filedialog.asksaveasfilename(initialdir=self.config_handler.get_last_used_directory(),
+                                                       title="Export {}".format(format),
+                                                       defaultextension="json" if format == "JSON" else "csv")
+
+        self.log.info("Exporting messages in {} format to {}".format(format,
+                                                                     output_location))
+        self.config_handler.save_last_used_directory(output_location)
+
+        try:
+            data = ""
+            if format == "JSON":
+                messages = list(self.subscribe_frame.messages.values())
+                for message in messages:
+                    message["payload"] = base64.b64encode(message["payload"]).decode("utf-8")
+                data = json.dumps(messages, indent=2)
+
+            if format == "CSV":
+                data = "ID,timestamp,date,time,subscription pattern,topic,QoS,retained,payload{}".format(os.linesep)
+                for message_id, message in self.subscribe_frame.messages.items():
+                    timestamp = message["timestamp"]
+                    datetime_object = datetime.fromtimestamp(timestamp)
+                    try:
+                        payload_decoded = str(message["payload"].decode("utf-8"))
+                    except Exception:
+                        payload_decoded = base64.b64encode(message["payload"]).decode("utf-8")
+                    data += '{},{},{},{},{},{},{},{},"{}"{}'.format(
+                        message_id,
+                        timestamp,
+                        datetime_object.strftime("%Y/%m/%d"),
+                        datetime_object.strftime("%H:%M:%S.%f"),
+                        message["subscription_pattern"],
+                        message["topic"],
+                        message["qos"],
+                        message["retained"],
+                        payload_decoded,
+                        os.linesep
+                    )
+
+            with open(output_location, "w") as outputfile:
+                outputfile.write(data)
+
+        except Exception as e:
+            self.log.exception("Failed to export message data", e, traceback.format_exc())
+            messagebox.showerror("Failed to export messages", "Failed to export messages: {} See log for details".format(e))
+        else:
+            self.log.info("Messages exported successfully")
+            messagebox.showinfo("Success", "Messages exported successfully")
+
+    def export_connection_config(self):
+        export_dialog = ConnectionConfigImportExport(self.root, self.icon, self.config_handler, self.log, False)
+
+    def import_connection_config(self):
+        import_dialog = ConnectionConfigImportExport(self.root, self.icon, self.config_handler, self.log, True)
+
+    def import_subscribe_publish(self):
+        import_dialog = SubscribePublishImportExport(self.root, self.icon, self.config_handler, self.log, True)
+
+    def export_subscribe_publish(self):
+        export_dialog = SubscribePublishImportExport(self.root, self.icon, self.config_handler, self.log, False)
+
 
 def main():
     app = App(root)

mqttk/config_handler.py

@@ -24,10 +24,36 @@
 from tkinter import messagebox
 from tkinter import filedialog
 import mqttk.mqtt_fx_config_parser as configparser
+from datetime import datetime
 
 LOAD = "load"
 SAVE = "save"
 
+DEFAULT_CONFIGURATION = {
+    "connections": {
+        "test.mosquitto.org": {
+            "connection_parameters": {
+            "broker_addr": "test.mosquitto.org",
+            "broker_port": "1883",
+            "client_id": "272fb6890d1c4eefac4f46b39deb83fb",
+            "user": "",
+            "pass": "",
+            "timeout": "10",
+            "keepalive": "60",
+            "mqtt_version": "3.1.1",
+            "ssl": "Disabled",
+            "ca_file": "",
+            "cl_cert": "",
+            "cl_key": ""
+            },
+        "subscriptions": {},
+        "publish_topics": [],
+        "stored_publishes": {},
+        "last_subscribe_used": "#"
+        }
+    }
+}
+
 
 class ConfigHandler:
     def __init__(self, logger):
@@ -39,7 +65,8 @@ def __init__(self, logger):
             "last_used_connection": connection name,
             "window_geometry: last used window geometry string,
             "autoscroll: true/false,
-            "last_used_decoder" = last used message decoder
+            "last_used_decoder": last used message decoder,
+            "last_used_directory": last used directory for browsing files
         }
 
         configuration_dict[connections] = {
@@ -81,6 +108,7 @@ def __init__(self, logger):
 
         """
         self.configuration_dict = {}
+        self.log_file = None
         self.wont_save = False
         self.first_start = True
         self.log = logger
@@ -102,6 +130,7 @@ def config_file_manager(self, action):
             appdata_dir = os.getenv('LOCALAPPDATA')
             config_dir = os.path.join(appdata_dir, "MQTTk")
             config_file = os.path.join(appdata_dir, "MQTTk", "MQTTk-config.json")
+            self.log_file = os.path.join(appdata_dir, "MQTTk", "MQTTk-log.txt")
 
         elif sys.platform.startswith("linux"):
             if self.first_start:
@@ -115,6 +144,7 @@ def config_file_manager(self, action):
             home_dir = str(Path.home())
             config_dir = os.path.join(home_dir, ".config", "MQTTk")
             config_file = os.path.join(home_dir, ".config", "MQTTk", "MQTTk-config.json")
+            self.log_file = os.path.join(home_dir, ".config", "MQTTk", "MQTTk-log.txt")
 
         elif sys.platform.startswith("darwin"):
             if self.first_start:
@@ -128,6 +158,7 @@ def config_file_manager(self, action):
             home_dir = str(Path.home())
             config_dir = os.path.join(home_dir, "Library", "ApplicationSupport", "MQTTk")
             config_file = os.path.join(home_dir, "Library", "ApplicationSupport", "MQTTk", "MQTTk-config.json")
+            self.log_file = os.path.join(home_dir, "Library", "ApplicationSupport", "MQTTk", "MQTTk-log.txt")
         else:
             self.log.warning("Unsupported platform detected. Configuration file won't be saved! Use this thing at your own risk :(")
             self.wont_save = True
@@ -138,11 +169,12 @@ def config_file_manager(self, action):
             return
 
         if not os.path.isfile(config_file):
+            self.configuration_dict = DEFAULT_CONFIGURATION
             self.first_start = True
             if not os.path.isdir(config_dir):
                 os.makedirs(config_dir)
             with open(config_file, "w") as configfile:
-                configfile.write("{}")
+                configfile.write(json.dumps(self.configuration_dict, indent=2))
 
         else:
             if action == LOAD:
@@ -311,3 +343,24 @@ def import_mqttfx_config(self):
         self.config_file_manager(SAVE)
         self.configuration_dict = response
         return True
+
+    def get_last_used_directory(self):
+        if "last_used_directory" not in self.configuration_dict:
+            self.configuration_dict["last_used_directory"] = Path.home()
+        if not os.path.isdir(self.configuration_dict["last_used_directory"]):
+            self.configuration_dict["last_used_directory"] = Path.home()
+        return self.configuration_dict["last_used_directory"]
+
+    def save_last_used_directory(self, directory):
+        head, tail = os.path.split(directory)
+        self.configuration_dict["last_used_directory"] = head
+        self.config_file_manager(SAVE)
+
+    def add_log_message(self, message):
+        if self.log_file is not None:
+            if not os.path.isfile(self.log_file):
+                with open(self.log_file, 'w') as logfile:
+                    logfile.write("New log file started {}{}".format(datetime.now().strftime("%Y/%m/%d, %H:%M:%S.%f"),
+                                                                     os.linesep))
+            with open(self.log_file, 'a') as logfile:
+                logfile.write(message)