From ba2c347f1b8f0d3bdf68f0abafb294af4fea54b4 Mon Sep 17 00:00:00 2001 From: Zhanyuan Zhang <32000378+zhanyuanucb@users.noreply.github.com> Date: Tue, 13 Dec 2022 06:10:54 +0000 Subject: [PATCH 1/4] test bloomz-560m --- examples/llm_serving/model/bloom_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_serving/model/bloom_model.py b/examples/llm_serving/model/bloom_model.py index 8cd65c00a..ce2249864 100644 --- a/examples/llm_serving/model/bloom_model.py +++ b/examples/llm_serving/model/bloom_model.py @@ -534,7 +534,7 @@ def __call__( def get_config(name, **kwargs): - if name == "bloom-560m": + if name in ["bloom-560m", "bloomz-560m"]: config = BloomConfig( hidden_size=1024, n_head=16, num_hidden_layers=24, pretraining_tp=1, use_cache=True From aabd15f59cbbe659fa3429f741c423c41d629882 Mon Sep 17 00:00:00 2001 From: Zhanyuan Zhang <32000378+zhanyuanucb@users.noreply.github.com> Date: Tue, 13 Dec 2022 08:23:35 +0000 Subject: [PATCH 2/4] test bloomz-7b1 --- examples/llm_serving/model/bloom_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_serving/model/bloom_model.py b/examples/llm_serving/model/bloom_model.py index ce2249864..b782368bc 100644 --- a/examples/llm_serving/model/bloom_model.py +++ b/examples/llm_serving/model/bloom_model.py @@ -554,7 +554,7 @@ def get_config(name, **kwargs): hidden_size=2560, n_head=32, num_hidden_layers=30, pretraining_tp=4, use_cache=True ) - elif name == "bloom-7b1": + elif name in ["bloom-7b1", "bloomz-7b1"]: config = BloomConfig( hidden_size=4096, n_head=32, num_hidden_layers=30, pretraining_tp=4, use_cache=True From e196638768392d22d55af41ada6f85b07abe69c4 Mon Sep 17 00:00:00 2001 From: Zhanyuan Zhang <32000378+zhanyuanucb@users.noreply.github.com> Date: Wed, 14 Dec 2022 18:25:13 +0000 Subject: [PATCH 3/4] add more bloomz support --- examples/llm_serving/model/bloom_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/llm_serving/model/bloom_model.py b/examples/llm_serving/model/bloom_model.py index b782368bc..6dc9db8db 100644 --- a/examples/llm_serving/model/bloom_model.py +++ b/examples/llm_serving/model/bloom_model.py @@ -539,17 +539,17 @@ def get_config(name, **kwargs): hidden_size=1024, n_head=16, num_hidden_layers=24, pretraining_tp=1, use_cache=True ) - elif name == "bloom-1b1": + elif name in ["bloom-1b1", "bloomz-1b1"]: config = BloomConfig( hidden_size=1536, n_head=16, num_hidden_layers=24, pretraining_tp=1, use_cache=True ) - elif name == "bloom-1b7": + elif name in ["bloom-1b7", "bloomz-1b7"]: config = BloomConfig( hidden_size=2048, n_head=16, num_hidden_layers=24, pretraining_tp=2, use_cache=True ) - elif name == "bloom-3b": + elif name in ["bloom-3b", "bloomz-3b"]: config = BloomConfig( hidden_size=2560, n_head=32, num_hidden_layers=30, pretraining_tp=4, use_cache=True From cbb5890da4160b8392afe497c2e5803f46dfb692 Mon Sep 17 00:00:00 2001 From: Zhanyuan Zhang <32000378+zhanyuanucb@users.noreply.github.com> Date: Tue, 3 Jan 2023 17:51:02 -0800 Subject: [PATCH 4/4] Update bloom_model.py --- examples/llm_serving/model/bloom_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/llm_serving/model/bloom_model.py b/examples/llm_serving/model/bloom_model.py index 6dc9db8db..e2673093c 100644 --- a/examples/llm_serving/model/bloom_model.py +++ b/examples/llm_serving/model/bloom_model.py @@ -559,7 +559,7 @@ def get_config(name, **kwargs): hidden_size=4096, n_head=32, num_hidden_layers=30, pretraining_tp=4, use_cache=True ) - elif name == "bloom": + elif name in ["bloom", "bloomz"]: config = BloomConfig( hidden_size=14336, n_head=112, num_hidden_layers=70, pretraining_tp=4, use_cache=True