From a53d4adf201f14dcc6c247565d5b603183d94fd1 Mon Sep 17 00:00:00 2001 From: Carlo Vallerga Date: Sun, 22 Jan 2023 07:49:30 +0100 Subject: [PATCH] Allow to generate more informative profile file name (#638) * Add SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME settings to generate more informative profile file name * Improve the algorithm that generate a file name from request path --- README.md | 5 +++++ project/tests/test_collector.py | 37 +++++++++++++++++++++++++++++++++ silk/collector.py | 31 ++++++++++++++++++++++++++- silk/config.py | 1 + 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78ee8eda..673dc524 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,11 @@ SILKY_PYTHON_PROFILER_RESULT_PATH = '/path/to/profiles/' A download button will become available with a binary `.prof` file for every request. This file can be used for further analysis using [snakeviz](https://github.com/jiffyclub/snakeviz) or other cProfile tools +To retrieve which endpoint generates a specific profile file it is possible to add a stub of the request path in the file name with the following: + +```python +SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True +``` Silk can also be used to profile specific blocks of code/functions. It provides a decorator and a context manager for this purpose. diff --git a/project/tests/test_collector.py b/project/tests/test_collector.py index 7123be1f..cd00a67b 100644 --- a/project/tests/test_collector.py +++ b/project/tests/test_collector.py @@ -2,6 +2,7 @@ from tests.util import DictStorage from silk.collector import DataCollector +from silk.config import SilkyConfig from .factories import RequestMinFactory @@ -45,3 +46,39 @@ def test_finalise(self): content = f.read() self.assertTrue(content) self.assertGreater(len(content), 0) + + def test_profile_file_name_with_disabled_extended_file_name(self): + SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = False + request_path = 'normal/uri/' + resulting_prefix = self._get_prof_file_name(request_path) + self.assertEqual(resulting_prefix, '') + + def test_profile_file_name_with_enabled_extended_file_name(self): + + SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True + request_path = 'normal/uri/' + resulting_prefix = self._get_prof_file_name(request_path) + self.assertEqual(resulting_prefix, 'normal_uri_') + + def test_profile_file_name_with_path_traversal_and_special_char(self): + SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True + request_path = 'spÉciàl/.././大/uri/@É/' + resulting_prefix = self._get_prof_file_name(request_path) + self.assertEqual(resulting_prefix, 'special_uri_e_') + + def test_profile_file_name_with_long_path(self): + SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME = True + request_path = 'long/path/' + 'a' * 100 + resulting_prefix = self._get_prof_file_name(request_path) + # the path is limited to 50 char plus the last `_` + self.assertEqual(len(resulting_prefix), 51) + + @classmethod + def _get_prof_file_name(cls, request_path: str) -> str: + request = RequestMinFactory() + request.path = request_path + DataCollector().configure(request) + DataCollector().finalise() + file_path = DataCollector().request.prof_file.name + filename = file_path.rsplit('/')[-1] + return filename.replace(f"{request.id}.prof", "") diff --git a/silk/collector.py b/silk/collector.py index ebfc9d4a..6c7f854b 100644 --- a/silk/collector.py +++ b/silk/collector.py @@ -2,6 +2,8 @@ import logging import marshal import pstats +import re +import unicodedata from io import StringIO from threading import local @@ -143,7 +145,8 @@ def finalise(self): self.request.pyprofile = profile_text if SilkyConfig().SILKY_PYTHON_PROFILER_BINARY: - file_name = self.request.prof_file.storage.get_available_name(f"{str(self.request.id)}.prof") + proposed_file_name = self._get_proposed_file_name() + file_name = self.request.prof_file.storage.get_available_name(proposed_file_name) with self.request.prof_file.storage.open(file_name, 'w+b') as f: marshal.dump(ps.stats, f) self.request.prof_file = f.name @@ -189,3 +192,29 @@ def finalise(self): def register_silk_query(self, *args): self.register_objects(TYP_SILK_QUERIES, *args) + + def _get_proposed_file_name(self) -> str: + """Retrieve the profile file name to be proposed to the storage""" + + if SilkyConfig().SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME: + slugified_path = slugify_path(self.request.path) + return f"{slugified_path}_{str(self.request.id)}.prof" + return f"{str(self.request.id)}.prof" + + +def slugify_path(request_path: str) -> str: + """ + Convert any characters not included in [a-zA-Z0-9_]) with a single underscore. + Convert to lowercase. Also strip any leading and trailing char that are not in the + accepted list + + Inspired from django slugify + """ + request_path = str(request_path) + request_path = ( + unicodedata.normalize("NFKD", request_path) + .encode("ascii", "ignore") + .decode("ascii") + ) + request_path = request_path.lower()[:50] + return re.sub(r'\W+', '_', request_path).strip('_') diff --git a/silk/config.py b/silk/config.py index d397c248..fddbf9e2 100644 --- a/silk/config.py +++ b/silk/config.py @@ -28,6 +28,7 @@ class SilkyConfig(metaclass=Singleton): 'SILKY_PYTHON_PROFILER': False, 'SILKY_PYTHON_PROFILER_FUNC': None, 'SILKY_STORAGE_CLASS': 'silk.storage.ProfilerResultStorage', + 'SILKY_PYTHON_PROFILER_EXTENDED_FILE_NAME': False, 'SILKY_MIDDLEWARE_CLASS': 'silk.middleware.SilkyMiddleware', 'SILKY_JSON_ENSURE_ASCII': True, 'SILKY_ANALYZE_QUERIES': False,