From 231523057a2e110e92ac927e6e0a0b393a930b9d Mon Sep 17 00:00:00 2001 From: bus Date: Fri, 16 Jun 2023 23:31:01 +0000 Subject: [PATCH] Improve JSON loading --- README.md | 5 +- README.rst | 3 +- cutelog/logger_tab.py | 4 +- cutelog/main_window.py | 131 +++++++++++++++++++++------ cutelog/resources/ui/about_dialog.ui | 11 ++- setup.py | 5 +- 6 files changed, 119 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index fef4153..72c60b7 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,9 @@ This is a graphical log viewer for Python's logging module. It can be targeted with a SocketHandler with no additional setup (see [Usage](#usage)). It can also be used from other languages or logging libraries with little effort (see the [Wiki](../../wiki/Creating-a-client-for-cutelog)). -For example, a Go library [gocutelog](https://github.com/busimus/gocutelog) shows how to enable +For example, a Go library [gocutelog](https://github.com/busimus/gocutelog) shows how to enable regular Go logging libraries to connect to cutelog. -This program is in beta, so please report bugs if you encounter them. - ## Features * Allows any number of simultaneous connections * Customizable look of log levels and columns, with presets for each @@ -65,6 +63,7 @@ Free software used: * [PyQt5](https://riverbankcomputing.com/software/pyqt/intro) - GPLv3 License, Copyright (c) 2019 Riverbank Computing Limited * [PySide2](https://wiki.qt.io/PySide2) - LGPLv3 License, Copyright (C) 2015 The Qt Company Ltd (http://www.qt.io/licensing/) * [QtPy](https://github.com/spyder-ide/qtpy) - MIT License, Copyright (c) 2011- QtPy contributors and others +* [jsonstream](https://github.com/Dunes/json_stream) - MIT License, Copyright (c) 2020 Dunes * [ion-icons](https://github.com/ionic-team/ionicons) - MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/) And thanks to [logview](https://pythonhosted.org/logview/) by Vinay Sajip for UI inspiration. diff --git a/README.rst b/README.rst index 516b01c..0d71466 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,6 @@ cutelog This is a graphical log viewer for Python's standard logging module. It can be targeted with a SocketHandler with no additional setup (see Usage_). - -The program is in beta: it's lacking some features and may be unstable, but it works. cutelog is cross-platform, although it's mainly written and optimized for Linux. Features @@ -17,6 +15,7 @@ Features * Customizable look of log levels and columns, with presets for each * Filtering based on level and namespace, as well as filtering by searching * Search through all records or only through filtered ones +* Display extra fields under the message with `Extra mode `_ * View exception tracebacks or messages in a separate window * Dark theme (with its own set of colors for levels) * Pop tabs out of the window, merge records of multiple tabs into one diff --git a/cutelog/logger_tab.py b/cutelog/logger_tab.py index 0c6a635..a354052 100644 --- a/cutelog/logger_tab.py +++ b/cutelog/logger_tab.py @@ -145,6 +145,8 @@ def __init__(self, logDict): self.created = logDict.get("created") if self.created is None: self.created = logDict.get("time") + if self.created is None: + self.created = logDict.get("_created") if self.created is None or type(self.created) not in (int, float): self.created = datetime.now().timestamp() @@ -741,7 +743,7 @@ def on_record(self, record): self.loggerTable.scrollToBottom() def add_conn_closed_record(self, conn): - record = LogRecord({'_cutelog': 'Connection {} closed'.format(conn.conn_id)}) + record = LogRecord({'_cutelog': 'Connection {} closed'.format(conn.conn_id), 'created': datetime.now().timestamp()}) self.on_record(record) def get_record(self, index): diff --git a/cutelog/main_window.py b/cutelog/main_window.py index 9cd2268..ed19cf6 100644 --- a/cutelog/main_window.py +++ b/cutelog/main_window.py @@ -1,6 +1,7 @@ -from qtpy.QtCore import QFile, Qt, QTextStream +from datetime import datetime +from qtpy.QtCore import QFile, Qt, QTextStream, QThread, Signal from qtpy.QtWidgets import (QFileDialog, QInputDialog, QMainWindow, QMenuBar, - QStatusBar, QTabWidget) + QStatusBar, QTabWidget, QProgressDialog) from .about_dialog import AboutDialog from .config import CONFIG @@ -15,6 +16,8 @@ class MainWindow(QMainWindow): + load_records = Signal(object) + def __init__(self, log, app, load_logfiles=[]): self.log = log.getChild('Main') self.app = app @@ -499,32 +502,28 @@ def open_load_records_dialog(self): d.open() def load_records(self, load_path): - import json - from os import path - - class RecordDecoder(json.JSONDecoder): - def __init__(self, *args, **kwargs): - json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs) - - def object_hook(self, obj): - if '_created' in obj: - obj['created'] = obj['_created'] - del obj['_created'] - record = LogRecord(obj) - del record._logDict['created'] - else: - record = LogRecord(obj) - return record - - name = path.basename(load_path) + from functools import partial + progress = QProgressDialog('Loading records...', 'Cancel', 0, 0, parent=self) + progress.setAutoClose(True) + progress.forceShow() + lt = LoadingThread(load_path, self.log, self) + lt.done_loading.connect(self.open_loaded_records) + lt.loading_error.connect(partial(show_critical_dialog, self, 'Error while loading records')) + progress.canceled.connect(lt.requestInterruption) + lt.finished.connect(progress.reset) + lt.start(priority=QThread.LowPriority) + + def open_loaded_records(self, loaded): + self.log.debug("Received loaded records") index = None - try: - with open(load_path, 'r') as f: - records = json.load(f, cls=RecordDecoder) - new_logger, index = self.create_logger(None, name) - new_logger.merge_with_records(records) - self.loggerTabWidget.setCurrentIndex(index) + if not loaded or not loaded[1]: + show_warning_dialog(self, "No records loaded", "No records could be loaded from the file") + return + logger_name, records = loaded + new_logger, index = self.create_logger(None, logger_name) + new_logger.merge_with_records(records) + self.loggerTabWidget.setCurrentIndex(index) self.set_status('Records have been loaded into "{}" tab'.format(new_logger.name)) except Exception as e: if index: @@ -540,7 +539,8 @@ def open_save_records_dialog(self): return d = QFileDialog(self) - d.selectFile(logger.name + '.log') + dt = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + d.selectFile("{}_{}.log".format(logger.name, dt)) d.setAcceptMode(QFileDialog.AcceptSave) d.setFileMode(QFileDialog.AnyFile) d.fileSelected.connect(partial(self.save_records, logger)) @@ -569,7 +569,7 @@ def __iter__(self): records = logger.record_model.records record_list = RecordList(records) with open(path, 'w') as f: - json.dump(record_list, f, indent=2) + json.dump(record_list, f, indent=1) self.set_status('Records have been saved to "{}"'.format(path)) except Exception as e: @@ -605,3 +605,78 @@ def shutdown(self): def signal_handler(self, *args): self.shutdown() + + +class LoadingThread(QThread): + done_loading = Signal(object) + loading_error = Signal(str) + + def __init__(self, load_path, log, parent=None): + super().__init__(parent) + self.load_path = load_path + self.log = log.getChild('LT') + + def run(self): + from os import path + self.log.debug('Starting loading thread') + records = [] + + try: + name = path.basename(self.load_path) + file = open(self.load_path, 'r') + except Exception as e: + text = "Error while opening the file: \n{}".format(e) + self.log.error(text, exc_info=True) + self.loading_error.emit(text) + return + try: + records = self.load(file) + except Exception as e: + text = "Error while loading records: \n{}".format(e) + self.log.error(text, exc_info=True) + self.loading_error.emit(text) + return + self.log.debug('Loading finished') + if not self.isInterruptionRequested(): + self.done_loading.emit((name, records)) + else: + self.log.warning("Loading was interrupted") + + def load(self, file): + try: + # Standard json module is the fastest when it comes to loading a single large object, + # and if it's not a valid JSON object it'll fail quickly and fall back to jsonstream + return self.load_native(file) + except Exception as e: + self.log.debug("Error while loading natively: {}".format(e), exc_info=True) + file.seek(0) + return self.load_stream(file) + + def load_native(self, file): + self.log.debug("Attempting to load natively") + import json + records = [] + rec_dicts = json.load(file) + for i, rec_dict in enumerate(rec_dicts): + records.append(LogRecord(rec_dict)) + if i % 10000 == 0: + if self.isInterruptionRequested(): + break + if len(records) == 0: + raise Exception("No records found") + return records + + def load_stream(self, file): + import jsonstream + self.log.debug("Attempting to load with jsonstream") + records = [] + for i, data in enumerate(jsonstream.load(file)): + if type(data) == list: + for rec in data: + records.append(LogRecord(rec)) + else: + records.append(LogRecord(data)) + if i % 1000 == 0: + if self.isInterruptionRequested(): + break + return records diff --git a/cutelog/resources/ui/about_dialog.ui b/cutelog/resources/ui/about_dialog.ui index 19e38ba..5b95a6b 100644 --- a/cutelog/resources/ui/about_dialog.ui +++ b/cutelog/resources/ui/about_dialog.ui @@ -71,14 +71,17 @@ p, li { white-space: pre-wrap; } <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;"> PySide2:</span></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"> LGPLv3 License, Copyright (C) 2015 The Qt Company Ltd (http://www.qt.io/licensing/)</p> <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">ion-icons:</span></p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/)</p> -<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">QtPy:</span></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2011- QtPy contributors and others</p> <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">jsonstream:</span></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2020 Dunes</p> +<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-weight:600;">ion-icons:</span></p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">MIT License, Copyright (c) 2015-present Ionic (http://ionic.io/)</p> +<p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">Thanks to logview by Vinay Sajip for UI inspiration.</p> -<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">And thanks to contributors to the cutelog project: Anil Berry, Thomas Hess</p> +<p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">And thanks to contributors to the cutelog project: Anil Berry, Thomas Hess, Ondrej Sienczak, Grzegorz Gajoch</p> <p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><br /></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:14pt;">Texts of all the mentioned licenses:</span></p> <p style=" margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;"><span style=" font-size:12pt;">The MIT License (MIT):</span></p> diff --git a/setup.py b/setup.py index 401cf7a..8c52841 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ from setuptools.command.build_py import build_py from setuptools.command.install import install -VERSION = '2.1.0' +VERSION = '2.2.0' def build_qt_resources(): @@ -64,7 +64,8 @@ def run(self): python_requires=">=3.5", install_requires=['PyQt5;platform_system=="Darwin"', # it's better to use distro-supplied 'PyQt5;platform_system=="Windows"', # PyQt package on Linux - 'QtPy'], + 'QtPy', + 'jsonstream==0.0.1'], classifiers=[ "Development Status :: 4 - Beta",