Commit 360ddbd 1 parent a480939 commit 360ddbd Copy full SHA for 360ddbd
File tree 1 file changed +3
-2
lines changed
1 file changed +3
-2
lines changed Original file line number Diff line number Diff line change 11
11
# - Server:
12
12
#
13
13
# ```bash
14
- # vllm serve mistralai/Pixtral-12B-2409 --tokenizer_mode mistral --limit_mm_per_prompt 'image=4' --max_num_batched_tokens 16384
14
+ # vllm serve mistralai/Pixtral-12B-2409 --tokenizer-mode mistral --limit-mm-per-prompt 'image=4' --max-model-len 16384
15
15
# ```
16
16
#
17
17
# - Client:
@@ -45,6 +45,7 @@ def run_simple_demo():
45
45
model_name = "mistralai/Pixtral-12B-2409"
46
46
sampling_params = SamplingParams (max_tokens = 8192 )
47
47
48
+ # Lower max_num_seqs or max_model_len on low-VRAM GPUs.
48
49
llm = LLM (model = model_name , tokenizer_mode = "mistral" )
49
50
50
51
prompt = "Describe this image in one sentence."
@@ -83,7 +84,7 @@ def run_advanced_demo():
83
84
model = model_name ,
84
85
tokenizer_mode = "mistral" ,
85
86
limit_mm_per_prompt = {"image" : max_img_per_msg },
86
- max_num_batched_tokens = max_img_per_msg * max_tokens_per_img ,
87
+ max_model_len = max_img_per_msg * max_tokens_per_img ,
87
88
)
88
89
89
90
prompt = "Describe the following image."
You can’t perform that action at this time.
0 commit comments