From eb50d2ee1dbd759e34e047416d30183cfc2f60e4 Mon Sep 17 00:00:00 2001 From: PaperMtn Date: Mon, 11 Nov 2024 20:41:57 +0000 Subject: [PATCH 1/2] Config and disable select pylint messages --- poetry.lock | 12 ++++++------ pyproject.toml | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/poetry.lock b/poetry.lock index cd53d8c..31ecab8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -305,13 +305,13 @@ files = [ [[package]] name = "packaging" -version = "24.1" +version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] @@ -526,13 +526,13 @@ requests = ">=2.0.1,<3.0.0" [[package]] name = "tomli" -version = "2.0.2" +version = "2.1.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" files = [ - {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, - {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, + {file = "tomli-2.1.0-py3-none-any.whl", hash = "sha256:a5c57c3d1c56f5ccdf89f6523458f60ef716e210fc47c4cfb188c5ba473e0391"}, + {file = "tomli-2.1.0.tar.gz", hash = "sha256:3f646cae2aec94e17d04973e4249548320197cfabdf130015d023de4b74d8ab8"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 28ae6de..c42b318 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,23 @@ pylint = "^3.3.1" [tool.poetry.scripts] gitlab-watchman = "gitlab_watchman:main" +[tool.pylint.messages_control] +max-line-length = 120 +max-attributes = 10 +max-args = 10 +disable = [ + "missing-module-docstring", + "too-few-public-methods", + "arguments-differ", + "logging-fstring-interpolation", + "no-else-return", + "no-else-raise", + "inconsistent-return-statements", + "broad-exception-caught", + "duplicate-code", +] + + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" \ No newline at end of file From a4fc204d35f034028893f7865f2c60b3932f4ee5 Mon Sep 17 00:00:00 2001 From: PaperMtn Date: Mon, 11 Nov 2024 20:42:29 +0000 Subject: [PATCH 2/2] Pylint based format updates and refactors --- src/gitlab_watchman/__init__.py | 2 + src/gitlab_watchman/__main__.py | 2 +- src/gitlab_watchman/clients/gitlab_client.py | 1 - src/gitlab_watchman/exceptions.py | 3 +- src/gitlab_watchman/loggers.py | 51 ++++++++++++-------- src/gitlab_watchman/models/blob.py | 2 +- src/gitlab_watchman/models/commit.py | 3 +- src/gitlab_watchman/models/file.py | 2 +- src/gitlab_watchman/models/group.py | 3 +- src/gitlab_watchman/models/issue.py | 3 +- src/gitlab_watchman/models/merge_request.py | 3 +- src/gitlab_watchman/models/milestone.py | 3 +- src/gitlab_watchman/models/note.py | 3 +- src/gitlab_watchman/models/project.py | 5 +- src/gitlab_watchman/models/signature.py | 15 +++--- src/gitlab_watchman/models/snippet.py | 5 +- src/gitlab_watchman/models/user.py | 2 +- src/gitlab_watchman/models/wiki_blob.py | 2 +- src/gitlab_watchman/signature_downloader.py | 1 + src/gitlab_watchman/utils.py | 1 + src/gitlab_watchman/watchman_processor.py | 42 +++++++++++++--- 21 files changed, 101 insertions(+), 53 deletions(-) diff --git a/src/gitlab_watchman/__init__.py b/src/gitlab_watchman/__init__.py index 267c1fd..9ac18c7 100644 --- a/src/gitlab_watchman/__init__.py +++ b/src/gitlab_watchman/__init__.py @@ -116,6 +116,8 @@ def validate_variables() -> bool: return True +# pylint: disable=too-many-locals, missing-function-docstring, global-variable-undefined +# pylint: disable=too-many-branches, disable=too-many-statements def main(): global OUTPUT_LOGGER try: diff --git a/src/gitlab_watchman/__main__.py b/src/gitlab_watchman/__main__.py index 427963d..8273c4f 100644 --- a/src/gitlab_watchman/__main__.py +++ b/src/gitlab_watchman/__main__.py @@ -1,3 +1,3 @@ from . import main -main() \ No newline at end of file +main() diff --git a/src/gitlab_watchman/clients/gitlab_client.py b/src/gitlab_watchman/clients/gitlab_client.py index 4191a8b..2b4b7a5 100644 --- a/src/gitlab_watchman/clients/gitlab_client.py +++ b/src/gitlab_watchman/clients/gitlab_client.py @@ -14,7 +14,6 @@ GitlabSearchError, GitlabHttpError ) - from gitlab_watchman.exceptions import ( GitLabWatchmanAuthenticationError, GitLabWatchmanGetObjectError, diff --git a/src/gitlab_watchman/exceptions.py b/src/gitlab_watchman/exceptions.py index 5d84338..a32d9a1 100644 --- a/src/gitlab_watchman/exceptions.py +++ b/src/gitlab_watchman/exceptions.py @@ -1,9 +1,8 @@ -from typing import Dict, Any - class GitLabWatchmanError(Exception): """ Base class for exceptions in GitLab Watchman. """ + class ElasticsearchMissingError(GitLabWatchmanError): """ Exception raised when Elasticsearch is not enabled on the instance. """ diff --git a/src/gitlab_watchman/loggers.py b/src/gitlab_watchman/loggers.py index 6a753e6..30032c1 100644 --- a/src/gitlab_watchman/loggers.py +++ b/src/gitlab_watchman/loggers.py @@ -16,20 +16,28 @@ class StdoutLogger: + """ Class to log to stdout """ def __init__(self, **kwargs): self.debug = kwargs.get('debug') self.print_header() init() + # pylint: disable=too-many-branches def log(self, - mes_type: str, + msg_level: str, message: Any, **kwargs) -> None: + """ Log to stdout + + Args: + msg_level: Level message to log + message: Message data to log + """ notify_type = kwargs.get('notify_type') scope = kwargs.get('scope') - if not self.debug and mes_type == 'DEBUG': + if not self.debug and msg_level == 'DEBUG': return if dataclasses.is_dataclass(message): @@ -44,7 +52,7 @@ def log(self, f' URL: {message.get("kas").get("externalUrl")} \n'\ f' VERSION: {message.get("kas").get("version")} \n' \ f' ENTERPRISE: {message.get("enterprise")}' - mes_type = 'INSTANCE' + msg_level = 'INSTANCE' if notify_type == "user": message = f'USER: \n' \ f' ID: {message.get("id")} \n' \ @@ -57,7 +65,7 @@ def log(self, f' CAN_CREATE_GROUP: {message.get("can_create_group")} \n'\ f' CAN_CREATE_PROJECT: {message.get("can_create_project")} \n' \ f' 2FA_ENABLED: {message.get("two_factor_enabled")}' - mes_type = 'USER' + msg_level = 'USER' if notify_type == "token": message = f'PERSONAL_ACCESS_TOKEN: \n' \ f' ID: {message.get("id")} \n' \ @@ -68,7 +76,7 @@ def log(self, f' LAST_USED_AT: {message.get("last_used_at")} \n' \ f' ACTIVE: {message.get("active")} \n'\ f' EXPIRY: {message.get("expires_at", "Never")}' - mes_type = 'WARNING' + msg_level = 'WARNING' if notify_type == "result": if scope == 'blobs': message = 'SCOPE: Blob' \ @@ -145,12 +153,12 @@ def log(self, f' URL: {message.get("snippet").get("web_url")} \n' \ f' POTENTIAL_SECRET: {message.get("match_string")} \n' \ f' -----' - mes_type = 'RESULT' + msg_level = 'RESULT' try: - self.log_to_stdout(message, mes_type) + self.log_to_stdout(message, msg_level) except Exception as e: print(e) - self.log_to_stdout(message, mes_type) + self.log_to_stdout(message, msg_level) def log_to_stdout(self, message: Any, @@ -236,7 +244,10 @@ def log_to_stdout(self, sys.exit(1) print('Formatting error') - def print_header(self) -> None: + @staticmethod + def print_header() -> None: + """ Prints the header for the logger""" + print(" ".ljust(79) + Style.BRIGHT) print(Fore.LIGHTRED_EX + Style.BRIGHT + @@ -265,6 +276,7 @@ def print_header(self) -> None: class JSONLogger(Logger): + """ Custom logger class for JSON logging""" def __init__(self, name: str = 'gitlab_watchman', **kwargs): super().__init__(name) self.notify_format = logging.Formatter( @@ -290,13 +302,13 @@ def __init__(self, name: str = 'gitlab_watchman', **kwargs): def log(self, level: str, - log_data: str or Dict, + msg: str or Dict, **kwargs): if level.upper() == 'NOTIFY': self.handler.setFormatter(self.notify_format) self.logger.info( json.dumps( - log_data, + msg, cls=EnhancedJSONEncoder), extra={ 'scope': kwargs.get('scope', ''), @@ -304,27 +316,28 @@ def log(self, 'severity': kwargs.get('severity', '')}) elif level.upper() == 'INFO': self.handler.setFormatter(self.info_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) elif level.upper() == 'DEBUG': self.handler.setFormatter(self.info_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) elif level.upper() == 'SUCCESS': self.handler.setFormatter(self.success_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) elif level.upper() == 'INSTANCE': self.handler.setFormatter(self.instance_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) elif level.upper() == 'USER': self.handler.setFormatter(self.user_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) elif level.upper() == 'TOKEN': self.handler.setFormatter(self.token_format) - self.logger.info(json.dumps(log_data)) + self.logger.info(json.dumps(msg)) else: self.handler.setFormatter(self.info_format) - self.logger.critical(log_data) + self.logger.critical(msg) +# pylint: disable=missing-class-docstring class IsDataclass(Protocol): __dataclass_fields__: ClassVar[Dict] @@ -360,4 +373,4 @@ def init_logger(logging_type: str, debug: bool) -> JSONLogger | StdoutLogger: if not logging_type or logging_type == 'stdout': return StdoutLogger(debug=debug) - return JSONLogger(debug=debug) \ No newline at end of file + return JSONLogger(debug=debug) diff --git a/src/gitlab_watchman/models/blob.py b/src/gitlab_watchman/models/blob.py index bd45ee6..d120549 100644 --- a/src/gitlab_watchman/models/blob.py +++ b/src/gitlab_watchman/models/blob.py @@ -2,7 +2,7 @@ @dataclass(slots=True) -class Blob(object): +class Blob: """ Class that defines Blob objects for GitLab blobs""" basename: str diff --git a/src/gitlab_watchman/models/commit.py b/src/gitlab_watchman/models/commit.py index 9b0d9e8..357b682 100644 --- a/src/gitlab_watchman/models/commit.py +++ b/src/gitlab_watchman/models/commit.py @@ -5,7 +5,8 @@ @dataclass(slots=True) -class Commit(object): +# pylint: disable=too-many-instance-attributes +class Commit: """ Class that defines File objects for GitLab files""" id: str diff --git a/src/gitlab_watchman/models/file.py b/src/gitlab_watchman/models/file.py index 5129251..6d5fcde 100644 --- a/src/gitlab_watchman/models/file.py +++ b/src/gitlab_watchman/models/file.py @@ -2,7 +2,7 @@ @dataclass(slots=True) -class File(object): +class File: """ Class that defines File objects for GitLab files""" file_name: str diff --git a/src/gitlab_watchman/models/group.py b/src/gitlab_watchman/models/group.py index e6c3046..c451da1 100644 --- a/src/gitlab_watchman/models/group.py +++ b/src/gitlab_watchman/models/group.py @@ -5,7 +5,8 @@ @dataclass(slots=True) -class Group(object): +# pylint: disable=too-many-instance-attributes +class Group: """ Class that defines User objects for GitLab groups""" id: str diff --git a/src/gitlab_watchman/models/issue.py b/src/gitlab_watchman/models/issue.py index ce49a22..dcc326a 100644 --- a/src/gitlab_watchman/models/issue.py +++ b/src/gitlab_watchman/models/issue.py @@ -6,7 +6,8 @@ @dataclass(slots=True) -class Issue(object): +# pylint: disable=too-many-instance-attributes +class Issue: """ Class that defines Issues objects for GitLab issues""" id: str diff --git a/src/gitlab_watchman/models/merge_request.py b/src/gitlab_watchman/models/merge_request.py index 8d4509d..d51b6b8 100644 --- a/src/gitlab_watchman/models/merge_request.py +++ b/src/gitlab_watchman/models/merge_request.py @@ -6,7 +6,8 @@ @dataclass(slots=True) -class MergeRequest(object): +# pylint: disable=too-many-instance-attributes +class MergeRequest: """ Class that defines MergeRequest objects for GitLab merge requests""" id: str diff --git a/src/gitlab_watchman/models/milestone.py b/src/gitlab_watchman/models/milestone.py index 1f03cd7..9a626ba 100644 --- a/src/gitlab_watchman/models/milestone.py +++ b/src/gitlab_watchman/models/milestone.py @@ -5,7 +5,8 @@ @dataclass(slots=True) -class Milestone(object): +# pylint: disable=too-many-instance-attributes +class Milestone: """ Class that defines Milestone objects for GitLab milestones""" id: str diff --git a/src/gitlab_watchman/models/note.py b/src/gitlab_watchman/models/note.py index 5815670..7e9cfa5 100644 --- a/src/gitlab_watchman/models/note.py +++ b/src/gitlab_watchman/models/note.py @@ -6,7 +6,8 @@ @dataclass(slots=True) -class Note(object): +# pylint: disable=too-many-instance-attributes +class Note: """ Class that defines User objects for GitLab notes""" id: str diff --git a/src/gitlab_watchman/models/project.py b/src/gitlab_watchman/models/project.py index ee43365..2f79006 100644 --- a/src/gitlab_watchman/models/project.py +++ b/src/gitlab_watchman/models/project.py @@ -7,7 +7,8 @@ @dataclass(slots=True) -class Namespace(object): +class Namespace: + """ Class that defines Namespace objects for GitLab Projects""" id: str name: str path: str @@ -20,7 +21,7 @@ class Namespace(object): @dataclass(slots=True) -class Project(object): +class Project: """ Class that defines User objects for GitLab projects""" id: str diff --git a/src/gitlab_watchman/models/signature.py b/src/gitlab_watchman/models/signature.py index 471d3d3..82d94b8 100644 --- a/src/gitlab_watchman/models/signature.py +++ b/src/gitlab_watchman/models/signature.py @@ -1,16 +1,17 @@ import datetime -from typing import Any, Dict +from typing import Any, Dict, List from dataclasses import dataclass -from typing import List @dataclass(slots=True) -class TestCases(object): +class TestCases: + """ Class that holds test cases for a signature """ match_cases: list fail_cases: list @dataclass(frozen=True, slots=True) +# pylint: disable=too-many-instance-attributes class Signature: """ Class that handles loaded signature objects. Signatures define what to search for in Slack and where to search for it. @@ -36,15 +37,13 @@ def __post_init__(self): raise TypeError(f'Expected `status` to be of type str, received {type(self.status).__name__}') if self.author and not isinstance(self.author, str): raise TypeError(f'Expected `author` to be of type str, received {type(self.author).__name__}') - if self.date and not (isinstance(self.date, datetime.date) - or isinstance(self.date, str) - or isinstance(self.date, datetime.datetime)): + if self.date and not isinstance(self.date, (datetime.date, datetime.datetime, str)): raise TypeError(f'Expected `date` to be of type str, received {type(self.date).__name__}') if self.version and not isinstance(self.version, str): raise TypeError(f'Expected `version` to be of type str, received {type(self.version).__name__}') if self.description and not isinstance(self.description, str): raise TypeError(f'Expected `description` to be of type str, received {type(self.description).__name__}') - if self.severity and not (isinstance(self.severity, int) or isinstance(self.severity, str)): + if self.severity and not isinstance(self.severity, (int, str)): raise TypeError(f'Expected `severity` to be of type int or str, received {type(self.severity).__name__}') if self.scope and not isinstance(self.scope, list): raise TypeError(f'Expected `scope` to be of type list, received {type(self.scope).__name__}') @@ -79,4 +78,4 @@ def create_from_dict(signature_dict: Dict[str, Any]) -> Signature: fail_cases=signature_dict.get('test_cases', {}).get('fail_cases') ), search_strings=signature_dict.get('watchman_apps', {}).get('gitlab', {}).get('search_strings'), - patterns=signature_dict.get('patterns')) \ No newline at end of file + patterns=signature_dict.get('patterns')) diff --git a/src/gitlab_watchman/models/snippet.py b/src/gitlab_watchman/models/snippet.py index eb9eff8..bfce73c 100644 --- a/src/gitlab_watchman/models/snippet.py +++ b/src/gitlab_watchman/models/snippet.py @@ -7,13 +7,14 @@ @dataclass(slots=True) -class File(object): +class File: + """ Class that defines File objects for GitLab snippets""" path: str raw_url: str @dataclass(slots=True) -class Snippet(object): +class Snippet: """ Class that defines User objects for GitLab snippets""" id: str diff --git a/src/gitlab_watchman/models/user.py b/src/gitlab_watchman/models/user.py index 970d206..dd4cf38 100644 --- a/src/gitlab_watchman/models/user.py +++ b/src/gitlab_watchman/models/user.py @@ -2,7 +2,7 @@ @dataclass(slots=True) -class User(object): +class User: """ Class that defines User objects for GitLab users""" id: str diff --git a/src/gitlab_watchman/models/wiki_blob.py b/src/gitlab_watchman/models/wiki_blob.py index dabbd85..ddeeb91 100644 --- a/src/gitlab_watchman/models/wiki_blob.py +++ b/src/gitlab_watchman/models/wiki_blob.py @@ -2,7 +2,7 @@ @dataclass(slots=True) -class WikiBlob(object): +class WikiBlob: """ Class that defines WikiBlob objects for GitLab blobs""" basename: str diff --git a/src/gitlab_watchman/signature_downloader.py b/src/gitlab_watchman/signature_downloader.py index 22c4764..b2cccb4 100644 --- a/src/gitlab_watchman/signature_downloader.py +++ b/src/gitlab_watchman/signature_downloader.py @@ -15,6 +15,7 @@ class SignatureDownloader: + """ A class for downloading and processing signature files from a GitHub repository. """ def __init__(self, logger: JSONLogger | StdoutLogger): """ Initializes a SignatureDownloader object. diff --git a/src/gitlab_watchman/utils.py b/src/gitlab_watchman/utils.py index 51839a7..4487926 100644 --- a/src/gitlab_watchman/utils.py +++ b/src/gitlab_watchman/utils.py @@ -7,6 +7,7 @@ class EnhancedJSONEncoder(json.JSONEncoder): + """ JSON Encoder that handles datetime and dataclass objects""" def default(self, o): if isinstance(o, datetime): return o.isoformat() diff --git a/src/gitlab_watchman/watchman_processor.py b/src/gitlab_watchman/watchman_processor.py index 2409548..790c77b 100644 --- a/src/gitlab_watchman/watchman_processor.py +++ b/src/gitlab_watchman/watchman_processor.py @@ -78,12 +78,12 @@ def find_group_owners(group_members: List[Dict]) -> List[Dict]: """ member_list = [] - for user in group_members: - if user.get('state') == 'active' and user.get('access_level') == 50: + for user_dict in group_members: + if user_dict.get('state') == 'active' and user_dict.get('access_level') == 50: member_list.append({ - 'user_id': user.get('id'), - 'name': user.get('name'), - 'username': user.get('username'), + 'user_id': user_dict.get('id'), + 'name': user_dict.get('name'), + 'username': user_dict.get('username'), 'access_level': 'Owner' }) @@ -91,6 +91,13 @@ def find_group_owners(group_members: List[Dict]) -> List[Dict]: def log_listener(log_queue: Queue, logging_type: str, debug: bool): + """ Listener for use in multiprocessing queued logging + + Args: + log_queue: Queue object + logging_type: Type of logging to use + debug: Whether to use debug level logging or not + """ log_handler = init_logger(logging_type, debug) while True: record = log_queue.get() @@ -108,6 +115,22 @@ def search(gitlab: GitLabAPIClient, scope: str, verbose: bool, timeframe: int = ALL_TIME) -> List[Dict] | None: + """ Use the appropriate search function to search GitLab based on the contents + of the signature file + + Args: + gitlab: GitLab API object + logging_type: Type of logging to use + log_handler: Logger object + debug: Whether to use debug level logging or not + sig: Signature object + scope: What sort of GitLab objects to search + verbose: Whether to use verbose logging + timeframe: Timeframe in seconds + Returns: + List of search results from GitLab API or None + + """ results = [] if logging_type == 'json': @@ -122,9 +145,11 @@ def search(gitlab: GitLabAPIClient, query_formatted = query.replace('"', '') if search_results: if logging_type == 'json': - log_queue.put(('INFO', f'{len(search_results)} {scope} found matching search term: {query_formatted}')) + log_queue.put(('INFO', f'{len(search_results)} {scope} ' + f'found matching search term: {query_formatted}')) else: - log_handler.log('INFO', f'{len(search_results)} {scope} found matching search term: {query_formatted}') + log_handler.log('INFO', f'{len(search_results)} {scope} ' + f'found matching search term: {query_formatted}') result = multiprocessing.Manager().list() @@ -514,7 +539,8 @@ def _snippet_worker(args: WorkerArgs) -> List[Dict]: try: snippet_object = snippet.create_from_dict(snippet_dict) if convert_to_epoch(snippet_object.created_at) > (now - args.timeframe) and \ - (args.regex.search(str(snippet_object.title)) or args.regex.search(str(snippet_object.description))): + (args.regex.search(str(snippet_object.title)) or + args.regex.search(str(snippet_object.description))): if args.regex.search(str(snippet_object.title)): match_string = args.regex.search(str(snippet_object.title)).group(0) else: