FEAT: Supports sglang backend

setup.cfg

@@ -96,6 +96,7 @@ all =
     orjson
     auto-gptq ; sys_platform!='darwin'
     optimum
+    sglang[all] ; sys_platform=='linux'
 intel =
     torch==2.1.0a0
     intel_extension_for_pytorch==2.1.10+xpu
@@ -118,6 +119,8 @@ transformers =
     peft
 vllm =
     vllm>=0.2.6
+sglang =
+    sglang[all]
 embedding =
     sentence-transformers>=2.3.1
 image =

xinference/constants.py

@@ -25,6 +25,7 @@
 XINFERENCE_ENV_HEALTH_CHECK_TIMEOUT = "XINFERENCE_HEALTH_CHECK_TIMEOUT"
 XINFERENCE_ENV_DISABLE_HEALTH_CHECK = "XINFERENCE_DISABLE_HEALTH_CHECK"
 XINFERENCE_ENV_DISABLE_VLLM = "XINFERENCE_DISABLE_VLLM"
+XINFERENCE_ENV_ENABLE_SGLANG = "XINFERENCE_ENABLE_SGLANG"
 
 
 def get_xinference_home() -> str:
@@ -64,3 +65,4 @@ def get_xinference_home() -> str:
     int(os.environ.get(XINFERENCE_ENV_DISABLE_HEALTH_CHECK, 0))
 )
 XINFERENCE_DISABLE_VLLM = bool(int(os.environ.get(XINFERENCE_ENV_DISABLE_VLLM, 0)))
+XINFERENCE_ENABLE_SGLANG = bool(int(os.environ.get(XINFERENCE_ENV_ENABLE_SGLANG, 0)))

xinference/model/llm/__init__.py

@@ -60,6 +60,7 @@ def _install():
     from .pytorch.qwen_vl import QwenVLChatModel
     from .pytorch.vicuna import VicunaPytorchChatModel
     from .pytorch.yi_vl import YiVLChatModel
+    from .sglang.core import SGLANGChatModel, SGLANGModel
     from .vllm.core import VLLMChatModel, VLLMModel
 
     # register llm classes.
@@ -79,6 +80,7 @@ def _install():
             CtransformersModel,
         ]
     )
+    LLM_CLASSES.extend([SGLANGModel, SGLANGChatModel])
     LLM_CLASSES.extend([VLLMModel, VLLMChatModel])
     LLM_CLASSES.extend(
         [

xinference/model/llm/sglang/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.