From bb11e9ddd0586da528f3aca19511f5e13846e3f9 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 17 Feb 2025 21:09:26 +0800 Subject: [PATCH 1/5] fix transformers dynamic module resolve with mp Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/model_executor/model_loader/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index dc620d4984a77..29d7360fe9e4b 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -50,9 +50,15 @@ def resolve_transformers_fallback(model_config: ModelConfig, custom_module = None auto_map = getattr(model_config.hf_config, "auto_map", None) if auto_map is not None and "AutoModel" in auto_map: - custom_module = get_class_from_dynamic_module( - model_config.hf_config.auto_map["AutoModel"], - model_config.model) + # NOTE(Isotr0py): we need to resolve all modules in auto_map across + # executor to avoid relative dynamic module imports error, otherwise + # some using related imported dynamic modules will be false-positive + # and failed to be imported in multiproc executor. + auto_modules = { + name: get_class_from_dynamic_module(module, model_config.model) + for name, module in auto_map.items() + } + custom_module = auto_modules["AutoModel"] # TODO(Isotr0py): Further clean up these raises. # perhaps handled them in _ModelRegistry._raise_for_unsupported? if model_config.model_impl == ModelImpl.TRANSFORMERS: From 0cac219196f3af9c0ae077dbac38f928affebb53 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 17 Feb 2025 21:41:38 +0800 Subject: [PATCH 2/5] better annotation Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/model_executor/model_loader/utils.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 29d7360fe9e4b..be812507ce6ec 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -47,8 +47,9 @@ def resolve_transformers_fallback(model_config: ModelConfig, for i, arch in enumerate(architectures): if arch == "TransformersModel": continue - custom_module = None - auto_map = getattr(model_config.hf_config, "auto_map", None) + custom_model_module = None + auto_map: Optional[dict[str, str]] = getattr(model_config.hf_config, + "auto_map", None) if auto_map is not None and "AutoModel" in auto_map: # NOTE(Isotr0py): we need to resolve all modules in auto_map across # executor to avoid relative dynamic module imports error, otherwise @@ -58,17 +59,17 @@ def resolve_transformers_fallback(model_config: ModelConfig, name: get_class_from_dynamic_module(module, model_config.model) for name, module in auto_map.items() } - custom_module = auto_modules["AutoModel"] + custom_model_module = auto_modules["AutoModel"] # TODO(Isotr0py): Further clean up these raises. # perhaps handled them in _ModelRegistry._raise_for_unsupported? if model_config.model_impl == ModelImpl.TRANSFORMERS: - if not is_transformers_impl_compatible(arch, custom_module): + if not is_transformers_impl_compatible(arch, custom_model_module): raise ValueError( f"The Transformers implementation of {arch} is not " "compatible with vLLM.") architectures[i] = "TransformersModel" if model_config.model_impl == ModelImpl.AUTO: - if not is_transformers_impl_compatible(arch, custom_module): + if not is_transformers_impl_compatible(arch, custom_model_module): raise ValueError( f"{arch} has no vLLM implementation and the Transformers " "implementation is not compatible with vLLM.") From 694ce2b92ac3318c85cfc2c491b500a8c401cf1d Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 17 Feb 2025 23:24:25 +0800 Subject: [PATCH 3/5] cleanup Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/model_executor/model_loader/utils.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index be812507ce6ec..e21b2e8334f39 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -47,19 +47,13 @@ def resolve_transformers_fallback(model_config: ModelConfig, for i, arch in enumerate(architectures): if arch == "TransformersModel": continue - custom_model_module = None - auto_map: Optional[dict[str, str]] = getattr(model_config.hf_config, - "auto_map", None) - if auto_map is not None and "AutoModel" in auto_map: - # NOTE(Isotr0py): we need to resolve all modules in auto_map across - # executor to avoid relative dynamic module imports error, otherwise - # some using related imported dynamic modules will be false-positive - # and failed to be imported in multiproc executor. - auto_modules = { - name: get_class_from_dynamic_module(module, model_config.model) - for name, module in auto_map.items() - } - custom_model_module = auto_modules["AutoModel"] + auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map", + None) or dict() + auto_modules = { + name: get_class_from_dynamic_module(module, model_config.model) + for name, module in auto_map.items() + } + custom_model_module = auto_modules.get("AutoModel") # TODO(Isotr0py): Further clean up these raises. # perhaps handled them in _ModelRegistry._raise_for_unsupported? if model_config.model_impl == ModelImpl.TRANSFORMERS: From 9263a11cba718e0b9ea0f29eda68a7b3fef3ca56 Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Mon, 17 Feb 2025 23:58:14 +0800 Subject: [PATCH 4/5] add comment back Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/model_executor/model_loader/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index e21b2e8334f39..06ce3e7d2c8bb 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -49,6 +49,10 @@ def resolve_transformers_fallback(model_config: ModelConfig, continue auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map", None) or dict() + # We need to get all dynamic modules from auto_map to make sure that + # they're all initialized properly. Otherwise, it will raise an error + # due to missing relative imported dynamic modules on spawn multiproc + # executor. auto_modules = { name: get_class_from_dynamic_module(module, model_config.model) for name, module in auto_map.items() From 65f81c767d8c841e544025aeb27a5cb8cc6b4f1a Mon Sep 17 00:00:00 2001 From: Isotr0py <2037008807@qq.com> Date: Tue, 18 Feb 2025 12:45:52 +0800 Subject: [PATCH 5/5] sort auto_map keys Signed-off-by: Isotr0py <2037008807@qq.com> --- vllm/model_executor/model_loader/utils.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 06ce3e7d2c8bb..9686231fb4bd1 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -49,13 +49,17 @@ def resolve_transformers_fallback(model_config: ModelConfig, continue auto_map: dict[str, str] = getattr(model_config.hf_config, "auto_map", None) or dict() - # We need to get all dynamic modules from auto_map to make sure that - # they're all initialized properly. Otherwise, it will raise an error - # due to missing relative imported dynamic modules on spawn multiproc - # executor. + # Make sure that config class is always initialized before model class, + # otherwise the model class won't be able to access the config class, + # the expected auto_map should have correct order like: + # "auto_map": { + # "AutoConfig": "--", + # "AutoModel": "--", + # "AutoModelFor": "--", + # }, auto_modules = { name: get_class_from_dynamic_module(module, model_config.model) - for name, module in auto_map.items() + for name, module in sorted(auto_map.items(), key=lambda x: x[0]) } custom_model_module = auto_modules.get("AutoModel") # TODO(Isotr0py): Further clean up these raises.