add cogvlm (#456)

paddlemix/examples/cogagent/README.md → paddlemix/examples/cogvlm/README.md

@@ -1,6 +1,15 @@
-# CogAgent
+## 1. CogVLM 模型介绍
 
-## 1. 模型介绍
+该模型是 [CogVLM](https://arxiv.org/abs/2311.03079) 的 paddle 实现。
+
+[CogVLM](https://arxiv.org/abs/2311.03079) 是一个强大的开源视觉语言模型（VLM）。CogVLM-17B拥有100亿的视觉参数和70亿的语言参数。
+
+CogVLM-17B在10个经典的跨模态基准测试中取得了最佳性能，包括 NoCaps, Flicker30k captioning, RefCOCO, RefCOCO+, RefCOCOg, Visual7W, GQA, ScienceQA, VizWiz VQA and TDIUC, 并在 VQAv2, OKVQA, TextVQA, COCO 字幕等方面排名第二., 超越或匹敌 PaLI-X 55B. CogVLM还可以和你聊关于图片的话题。
+
+本仓库提供paddle版本的 cogvlm-chat 模型
+
+
+## 2. CogAgent 模型介绍
 
 该模型是 [CogAgent](https://arxiv.org/abs/2312.08914) 的 paddle 实现。
 
@@ -20,19 +29,19 @@ CogAgent-18B在9个经典的跨模态基准测试中实现了最先进的全能
 
 本仓库提供paddle版本的 cogagent-chat 模型
 
-## 2. 环境准备
+## 3. 环境准备
 
 1） [安装PaddleNLP](https://github.com/PaddlePaddle/PaddleNLP?tab=readme-ov-file#%E5%AE%89%E8%A3%85)
 
 2）[安装 PaddleMix 环境依赖包](https://github.com/PaddlePaddle/PaddleMIX/tree/b4f97ff859e1964c839fc5fab94f7ba63b1e5959?tab=readme-ov-file#%E5%AE%89%E8%A3%85)
 
-## 3. 快速开始
+## 4. 快速开始
 完成环境准备后，我们目前提供多轮对话方式使用：
 
 ```bash
-python paddlemix/examples/cogagent/chat_demo.py \
---from_pretrained "THUDM/cogagent-chat"
+python paddlemix/examples/cogvlm/chat_demo.py \
+--from_pretrained "THUDM/cogvlm-chat"
 ```
 
 可配置参数说明：
-  * `from_pretrained`: 指定CogAgent的模型名字或权重路径以及tokenizer，默认 THUDM/cogagent-chat
+  * `from_pretrained`: 指定cogvlm的模型名字或权重路径以及tokenizer，默认 THUDM/cogvlm-chat

paddlemix/examples/cogagent/chat_demo.py → paddlemix/examples/cogvlm/chat_demo.py

@@ -31,7 +31,7 @@
 
 parser = argparse.ArgumentParser()
 
-parser.add_argument("--from_pretrained", type=str, default="THUDM/cogagent-chat", help="pretrained ckpt and tokenizer")
+parser.add_argument("--from_pretrained", type=str, default="THUDM/cogvlm-chat", help="pretrained ckpt and tokenizer")
 args = parser.parse_args()
 MODEL_PATH = args.from_pretrained
 TOKENIZER_PATH = MODEL_PATH

paddlemix/models/__init__.py

@@ -16,6 +16,8 @@
 from .audioldm2.configuration import *
 from .audioldm2.modeling import *
 from .blip2.modeling import *
+from .cogvlm.configuration import *
+from .cogvlm.modeling import *
 from .imagebind.modeling import *
 from .imagebind.multimodal_preprocessors import *
 from .llava import *

paddlemix/models/cogvlm/__init__.py

@@ -0,0 +1,16 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .configuration import *
+from .modeling import *

paddlemix/models/cogagent/configuration.py → paddlemix/models/cogvlm/configuration.py

@@ -17,7 +17,7 @@
 from paddlenlp import transformers
 
 
-class CogAgentConfig(transformers.PretrainedConfig):
+class CogModelConfig(transformers.PretrainedConfig):
     _auto_class = "AutoConfig"
 
     def __init__(
@@ -40,6 +40,7 @@ def __init__(
         eos_token_id=2,
         tie_word_embeddings=False,
         use_cache=True,
+        model_type="cogagent",
         **kwargs
     ):
         self.hidden_size = hidden_size
@@ -56,6 +57,7 @@ def __init__(
         self.hidden_act = hidden_act
         self.template_version = template_version
         self.use_cache = use_cache
+        self.model_type = model_type
         super().__init__(
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,