Improve JSON loading

README.md

@@ -5,11 +5,9 @@ This is a graphical log viewer for Python's logging module.
 It can be targeted with a SocketHandler with no additional setup (see [Usage](#usage)).
 
 It can also be used from other languages or logging libraries with little effort (see the [Wiki](../../wiki/Creating-a-client-for-cutelog)).
-For example, a Go library [gocutelog](https://github.com/busimus/gocutelog) shows how to enable 
+For example, a Go library [gocutelog](https://github.com/busimus/gocutelog) shows how to enable
 regular Go logging libraries to connect to cutelog.
 
-This program is in beta, so please report bugs if you encounter them.
-
 ## Features
 * Allows any number of simultaneous connections
 * Customizable look of log levels and columns, with presets for each
@@ -65,6 +63,7 @@ Free software used:
     * [PyQt5](https://riverbankcomputing.com/software/pyqt/intro) - GPLv3 License, Copyright (c) 2019 Riverbank Computing Limited <info@riverbankcomputing.com>
     * [PySide2](https://wiki.qt.io/PySide2) - LGPLv3 License, Copyright (C) 2015 The Qt Company Ltd (http://www.qt.io/licensing/)
 * [QtPy](https://github.com/spyder-ide/qtpy) - MIT License, Copyright (c) 2011- QtPy contributors and others
+* [jsonstream](https://github.com/Dunes/json_stream) - MIT License, Copyright (c) 2020 Dunes
 * [ion-icons](https://github.com/ionic-team/ionicons) - MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/)
 
 And thanks to [logview](https://pythonhosted.org/logview/) by Vinay Sajip for UI inspiration.

README.rst

@@ -7,8 +7,6 @@ cutelog
 
 This is a graphical log viewer for Python's standard logging module.
 It can be targeted with a SocketHandler with no additional setup (see Usage_).
-
-The program is in beta: it's lacking some features and may be unstable, but it works.
 cutelog is cross-platform, although it's mainly written and optimized for Linux.
 
 Features
@@ -17,6 +15,7 @@ Features
 * Customizable look of log levels and columns, with presets for each
 * Filtering based on level and namespace, as well as filtering by searching
 * Search through all records or only through filtered ones
+* Display extra fields under the message with `Extra mode <https://github.com/busimus/cutelog/wiki/Creating-a-client-for-cutelog#extra-mode>`_
 * View exception tracebacks or messages in a separate window
 * Dark theme (with its own set of colors for levels)
 * Pop tabs out of the window, merge records of multiple tabs into one

cutelog/logger_tab.py

@@ -145,6 +145,8 @@ def __init__(self, logDict):
         self.created = logDict.get("created")
         if self.created is None:
             self.created = logDict.get("time")
+            if self.created is None:
+                self.created = logDict.get("_created")
         if self.created is None or type(self.created) not in (int, float):
             self.created = datetime.now().timestamp()
 
@@ -741,7 +743,7 @@ def on_record(self, record):
             self.loggerTable.scrollToBottom()
 
     def add_conn_closed_record(self, conn):
-        record = LogRecord({'_cutelog': 'Connection {} closed'.format(conn.conn_id)})
+        record = LogRecord({'_cutelog': 'Connection {} closed'.format(conn.conn_id), 'created': datetime.now().timestamp()})
         self.on_record(record)
 
     def get_record(self, index):

cutelog/main_window.py

@@ -1,6 +1,7 @@
-from qtpy.QtCore import QFile, Qt, QTextStream
+from datetime import datetime
+from qtpy.QtCore import QFile, Qt, QTextStream, QThread, Signal
 from qtpy.QtWidgets import (QFileDialog, QInputDialog, QMainWindow, QMenuBar,
-                            QStatusBar, QTabWidget)
+                            QStatusBar, QTabWidget, QProgressDialog)
 
 from .about_dialog import AboutDialog
 from .config import CONFIG
@@ -15,6 +16,8 @@
 
 class MainWindow(QMainWindow):
 
+    load_records = Signal(object)
+
     def __init__(self, log, app, load_logfiles=[]):
         self.log = log.getChild('Main')
         self.app = app
@@ -499,32 +502,28 @@ def open_load_records_dialog(self):
         d.open()
 
     def load_records(self, load_path):
-        import json
-        from os import path
-
-        class RecordDecoder(json.JSONDecoder):
-            def __init__(self, *args, **kwargs):
-                json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
-
-            def object_hook(self, obj):
-                if '_created' in obj:
-                    obj['created'] = obj['_created']
-                    del obj['_created']
-                    record = LogRecord(obj)
-                    del record._logDict['created']
-                else:
-                    record = LogRecord(obj)
-                return record
-
-        name = path.basename(load_path)
+        from functools import partial
+        progress = QProgressDialog('Loading records...', 'Cancel', 0, 0, parent=self)
+        progress.setAutoClose(True)
+        progress.forceShow()
+        lt = LoadingThread(load_path, self.log, self)
+        lt.done_loading.connect(self.open_loaded_records)
+        lt.loading_error.connect(partial(show_critical_dialog, self, 'Error while loading records'))
+        progress.canceled.connect(lt.requestInterruption)
+        lt.finished.connect(progress.reset)
+        lt.start(priority=QThread.LowPriority)
+
+    def open_loaded_records(self, loaded):
+        self.log.debug("Received loaded records")
         index = None
-
         try:
-            with open(load_path, 'r') as f:
-                records = json.load(f, cls=RecordDecoder)
-                new_logger, index = self.create_logger(None, name)
-                new_logger.merge_with_records(records)
-                self.loggerTabWidget.setCurrentIndex(index)
+            if not loaded or not loaded[1]:
+                show_warning_dialog(self, "No records loaded", "No records could be loaded from the file")
+                return
+            logger_name, records = loaded
+            new_logger, index = self.create_logger(None, logger_name)
+            new_logger.merge_with_records(records)
+            self.loggerTabWidget.setCurrentIndex(index)
             self.set_status('Records have been loaded into "{}" tab'.format(new_logger.name))
         except Exception as e:
             if index:
@@ -540,7 +539,8 @@ def open_save_records_dialog(self):
             return
 
         d = QFileDialog(self)
-        d.selectFile(logger.name + '.log')
+        dt = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+        d.selectFile("{}_{}.log".format(logger.name, dt))
         d.setAcceptMode(QFileDialog.AcceptSave)
         d.setFileMode(QFileDialog.AnyFile)
         d.fileSelected.connect(partial(self.save_records, logger))
@@ -569,7 +569,7 @@ def __iter__(self):
             records = logger.record_model.records
             record_list = RecordList(records)
             with open(path, 'w') as f:
-                json.dump(record_list, f, indent=2)
+                json.dump(record_list, f, indent=1)
             self.set_status('Records have been saved to "{}"'.format(path))
 
         except Exception as e:
@@ -605,3 +605,78 @@ def shutdown(self):
 
     def signal_handler(self, *args):
         self.shutdown()
+
+
+class LoadingThread(QThread):
+    done_loading = Signal(object)
+    loading_error = Signal(str)
+
+    def __init__(self, load_path, log, parent=None):
+        super().__init__(parent)
+        self.load_path = load_path
+        self.log = log.getChild('LT')
+
+    def run(self):
+        from os import path
+        self.log.debug('Starting loading thread')
+        records = []
+
+        try:
+            name = path.basename(self.load_path)
+            file = open(self.load_path, 'r')
+        except Exception as e:
+            text = "Error while opening the file: \n{}".format(e)
+            self.log.error(text, exc_info=True)
+            self.loading_error.emit(text)
+            return
+        try:
+            records = self.load(file)
+        except Exception as e:
+            text = "Error while loading records: \n{}".format(e)
+            self.log.error(text, exc_info=True)
+            self.loading_error.emit(text)
+            return
+        self.log.debug('Loading finished')
+        if not self.isInterruptionRequested():
+            self.done_loading.emit((name, records))
+        else:
+            self.log.warning("Loading was interrupted")
+
+    def load(self, file):
+        try:
+            # Standard json module is the fastest when it comes to loading a single large object,
+            # and if it's not a valid JSON object it'll fail quickly and fall back to jsonstream
+            return self.load_native(file)
+        except Exception as e:
+            self.log.debug("Error while loading natively: {}".format(e), exc_info=True)
+        file.seek(0)
+        return self.load_stream(file)
+
+    def load_native(self, file):
+        self.log.debug("Attempting to load natively")
+        import json
+        records = []
+        rec_dicts = json.load(file)
+        for i, rec_dict in enumerate(rec_dicts):
+            records.append(LogRecord(rec_dict))
+            if i % 10000 == 0:
+                if self.isInterruptionRequested():
+                    break
+        if len(records) == 0:
+            raise Exception("No records found")
+        return records
+
+    def load_stream(self, file):
+        import jsonstream
+        self.log.debug("Attempting to load with jsonstream")
+        records = []
+        for i, data in enumerate(jsonstream.load(file)):
+            if type(data) == list:
+                for rec in data:
+                    records.append(LogRecord(rec))
+            else:
+                records.append(LogRecord(data))
+            if i % 1000 == 0:
+                if self.isInterruptionRequested():
+                    break
+        return records

cutelog/resources/ui/about_dialog.ui

@@ -71,14 +71,17 @@ p, li { white-space: pre-wrap; }
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">    PySide2:</span></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">    LGPLv3 License, Copyright (C) 2015 The Qt Company Ltd (http://www.qt.io/licensing/)</p>
 <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
-<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">ion-icons:</span></p>
-<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/)</p>
-<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">QtPy:</span></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2011- QtPy contributors and others</p>
 <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
+<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">jsonstream:</span></p>
+<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2020 Dunes</p>
+<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
+<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">ion-icons:</span></p>
+<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/)</p>
+<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">Thanks to logview by Vinay Sajip for UI inspiration.</p>
-<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">And thanks to contributors to the cutelog project: Anil Berry, Thomas Hess</p>
+<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">And thanks to contributors to the cutelog project: Anil Berry, Thomas Hess, Ondrej Sienczak, Grzegorz Gajoch</p>
 <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:14pt;">Texts of all the mentioned licenses:</span></p>
 <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:12pt;">The MIT License (MIT):</span></p>

setup.py

@@ -6,7 +6,7 @@
 from setuptools.command.build_py import build_py
 from setuptools.command.install import install
 
-VERSION = '2.1.0'
+VERSION = '2.2.0'
 
 
 def build_qt_resources():
@@ -64,7 +64,8 @@ def run(self):
     python_requires=">=3.5",
     install_requires=['PyQt5;platform_system=="Darwin"',  # it's better to use distro-supplied
                       'PyQt5;platform_system=="Windows"',  # PyQt package on Linux
-                      'QtPy'],
+                      'QtPy',
+                      'jsonstream==0.0.1'],
 
     classifiers=[
         "Development Status :: 4 - Beta",