From 781b874e27fbc8f168157318d63a59a41c1f3ebf Mon Sep 17 00:00:00 2001 From: Travis Johnson Date: Mon, 18 Nov 2024 11:06:45 -0700 Subject: [PATCH 1/2] fix: token counter variable should be int not float Signed-off-by: Travis Johnson --- vllm/engine/llm_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py index 9a2d73a020c8f..e72dc81f35b67 100644 --- a/vllm/engine/llm_engine.py +++ b/vllm/engine/llm_engine.py @@ -1716,7 +1716,7 @@ def _get_stats(self, # not counted (to avoid double counting) actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore - num_generation_tokens_from_prefill_groups = 0. + num_generation_tokens_from_prefill_groups = 0 # NOTE: if scheduler_outputs.num_prefill_groups > 0 and # the len of scheduler_outputs.scheduled_seq_groups is != # scheduler_outputs.num_prefill_groups, this means that From 338740afea2d001a7409f266b930cb4d88fb5ce0 Mon Sep 17 00:00:00 2001 From: Travis Johnson Date: Mon, 18 Nov 2024 11:19:39 -0700 Subject: [PATCH 2/2] add check to prevent exception from negative counter Signed-off-by: Travis Johnson --- vllm/engine/metrics.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/engine/metrics.py b/vllm/engine/metrics.py index e896bcdded2d1..47472c274ccb6 100644 --- a/vllm/engine/metrics.py +++ b/vllm/engine/metrics.py @@ -512,6 +512,11 @@ def _log_gauge(self, gauge, data: Union[int, float]) -> None: def _log_counter(self, counter, data: Union[int, float]) -> None: # Convenience function for logging to counter. + # Prevent ValueError from negative increment + if data < 0: + logger.warning("Skipping negative increment of %g to %s", data, + counter) + return counter.labels(**self.labels).inc(data) def _log_counter_labels(self, counter, data: CollectionsCounter,