From 4224aa90a4611baffddf7dc55aa3e3b90cea34e2 Mon Sep 17 00:00:00 2001 From: AnsibleGuy Date: Sun, 6 Oct 2024 20:31:46 +0200 Subject: [PATCH] cleanup job-status if thread stopped --- CHANGELOG.md | 8 ++++++++ src/ansibleguy-webui/aw/execute/play.py | 2 +- src/ansibleguy-webui/aw/execute/play_util.py | 5 +++-- src/ansibleguy-webui/aw/execute/repository.py | 3 ++- src/ansibleguy-webui/aw/execute/threader.py | 7 ++++++- src/ansibleguy-webui/aw/execute/util.py | 7 +++++-- src/ansibleguy-webui/aw/model/base.py | 4 +++- src/ansibleguy-webui/db.py | 12 ++---------- 8 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c37639a..7e7f215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ ## Version 0 +### 0.0.25 + +* Handle edge-case error of schedule +* Handle jobs without credentials +* Cleanup execution status if thread stopped + +---- + ### 0.0.24-3 * Disable data-refresh if in background diff --git a/src/ansibleguy-webui/aw/execute/play.py b/src/ansibleguy-webui/aw/execute/play.py index a2d9384..1c7b36c 100644 --- a/src/ansibleguy-webui/aw/execute/play.py +++ b/src/ansibleguy-webui/aw/execute/play.py @@ -63,7 +63,7 @@ def _cancel_job() -> bool: runner_cleanup(execution=execution, path_run=path_run, exec_repo=exec_repo) Alert(job=job, execution=execution).go() - except (OSError, AnsibleConfigError) as err: + except (OSError, AnsibleConfigError, ValueError, AttributeError, IndexError, KeyError) as err: tb = traceback.format_exc(limit=1024) failure( execution=execution, exec_repo=exec_repo, path_run=path_run, result=result, diff --git a/src/ansibleguy-webui/aw/execute/play_util.py b/src/ansibleguy-webui/aw/execute/play_util.py index ca8096f..310afe3 100644 --- a/src/ansibleguy-webui/aw/execute/play_util.py +++ b/src/ansibleguy-webui/aw/execute/play_util.py @@ -21,6 +21,7 @@ from aw.utils.debug import log from aw.execute.repository import ExecuteRepository from aw.execute.play_credentials import get_runner_credential_args, get_credentials_to_use +from aw.model.base import JOB_EXEC_STATUS_FAILED # see: https://ansible.readthedocs.io/projects/runner/en/latest/intro/ @@ -263,7 +264,7 @@ def parse_run_result(execution: JobExecution, result: JobExecutionResult, runner any_task_failed = _run_stats(runner=runner, result=result) if runner.errored or runner.timed_out or runner.rc != 0 or any_task_failed: - update_status(execution, status='Failed') + update_status(execution, status=JOB_EXEC_STATUS_FAILED) else: status = 'Finished' @@ -277,7 +278,7 @@ def failure( execution: JobExecution, exec_repo: ExecuteRepository, path_run: Path, result: JobExecutionResult, error_s: str, error_m: str ): - update_status(execution, status='Failed') + update_status(execution, status=JOB_EXEC_STATUS_FAILED) job_error = JobError( short=error_s, med=error_m, diff --git a/src/ansibleguy-webui/aw/execute/repository.py b/src/ansibleguy-webui/aw/execute/repository.py index f4b89d4..9267a8b 100644 --- a/src/ansibleguy-webui/aw/execute/repository.py +++ b/src/ansibleguy-webui/aw/execute/repository.py @@ -14,6 +14,7 @@ from aw.utils.handlers import AnsibleRepositoryError from aw.model.repository import Repository from aw.base import USERS +from aw.model.base import JOB_EXEC_STATUS_FAILED class ExecuteRepository: @@ -113,7 +114,7 @@ def create_or_update_repository(self): def _error(self, msg: str): write_file_0640(file=self.repository.log_stderr, content=msg) - update_status(self.repository, status='Failed') + update_status(self.repository, status=JOB_EXEC_STATUS_FAILED) raise AnsibleRepositoryError(msg).with_traceback(None) from None def get_path_run_repo(self) -> Path: diff --git a/src/ansibleguy-webui/aw/execute/threader.py b/src/ansibleguy-webui/aw/execute/threader.py index 7806698..4178893 100644 --- a/src/ansibleguy-webui/aw/execute/threader.py +++ b/src/ansibleguy-webui/aw/execute/threader.py @@ -11,13 +11,15 @@ from aw.execute.play import ansible_playbook from aw.utils.handlers import AnsibleConfigError, AnsibleRepositoryError from aw.utils.util import get_next_cron_execution_sec, get_next_cron_execution_str, is_set +from aw.execute.util import update_status +from aw.model.base import JOB_EXEC_STATUS_ACTIVE, JOB_EXEC_STATUS_FAILED class Workload(Thread): FAIL_SLEEP = 5 MAX_CONFIG_INVALID = 3 - def __init__(self, job: Job, manager, name: str, execution: JobExecution, once: bool = False, daemon: bool = True): + def __init__(self, job: Job, manager, name: str, execution: (JobExecution, None), once: bool = False, daemon: bool = True): Thread.__init__(self, daemon=daemon, name=name) self.job = job self.execution = execution @@ -44,6 +46,9 @@ def stop(self) -> bool: # 'cannot join current thread' pass + if self.execution is not None and self.execution.status in JOB_EXEC_STATUS_ACTIVE: + update_status(self.execution, status=JOB_EXEC_STATUS_FAILED) + log(f"Stopped thread {self.log_name_debug}", level=4) self.started = False self.stopped = True diff --git a/src/ansibleguy-webui/aw/execute/util.py b/src/ansibleguy-webui/aw/execute/util.py index 0530028..61136ae 100644 --- a/src/ansibleguy-webui/aw/execute/util.py +++ b/src/ansibleguy-webui/aw/execute/util.py @@ -50,8 +50,11 @@ def decode_job_env_vars(env_vars_csv: str, src: str) -> dict: return {} -def update_status(obj: (JobExecution, Repository), status: str): - obj.status = obj.status_id_from_name(status) +def update_status(obj: (JobExecution, Repository), status: (str, int)): + if isinstance(status, str): + status = obj.status_id_from_name(status) + + obj.status = status obj.save() diff --git a/src/ansibleguy-webui/aw/model/base.py b/src/ansibleguy-webui/aw/model/base.py index 9fb559e..6a49128 100644 --- a/src/ansibleguy-webui/aw/model/base.py +++ b/src/ansibleguy-webui/aw/model/base.py @@ -6,15 +6,17 @@ ) DEFAULT_NONE = {'null': True, 'default': None, 'blank': True} JOB_EXEC_STATUS_SUCCESS = 4 +JOB_EXEC_STATUS_FAILED = 3 CHOICES_JOB_EXEC_STATUS = [ (0, 'Waiting'), (1, 'Starting'), (2, 'Running'), - (3, 'Failed'), + (JOB_EXEC_STATUS_FAILED, 'Failed'), (JOB_EXEC_STATUS_SUCCESS, 'Finished'), (5, 'Stopping'), (6, 'Stopped'), ] +JOB_EXEC_STATUS_ACTIVE = [0, 1, 2, 5] class BareModel(models.Model): diff --git a/src/ansibleguy-webui/db.py b/src/ansibleguy-webui/db.py index 753124a..6536476 100644 --- a/src/ansibleguy-webui/db.py +++ b/src/ansibleguy-webui/db.py @@ -223,15 +223,7 @@ def create_manager_groups(): def cleanup_job_stati(): - from aw.model.base import CHOICES_JOB_EXEC_STATUS + from aw.model.base import JOB_EXEC_STATUS_ACTIVE, JOB_EXEC_STATUS_FAILED from aw.model.job import JobExecution - from aw.utils.util import get_choice_key_by_value - bad_stati = [ - get_choice_key_by_value(choices=CHOICES_JOB_EXEC_STATUS, find='Waiting'), - get_choice_key_by_value(choices=CHOICES_JOB_EXEC_STATUS, find='Running'), - get_choice_key_by_value(choices=CHOICES_JOB_EXEC_STATUS, find='Stopping'), - ] - set_status = get_choice_key_by_value(choices=CHOICES_JOB_EXEC_STATUS, find='Failed') - - JobExecution.objects.filter(status__in=bad_stati).update(status=set_status) + JobExecution.objects.filter(status__in=JOB_EXEC_STATUS_ACTIVE).update(status=JOB_EXEC_STATUS_FAILED)