许可证: MIT
语言支持:

• 英语
• 德语
• 法语
• 意大利语
• 葡萄牙语
• 印地语
• 西班牙语
• 泰语
  基础模型:
• Qwen/Qwen2.5-Omni-7B
  任务标签: 任意到任意
  标签:
• GPTQ量化模型
• FunAGI
• Qwen
• int4量化

该模型是基于GPTQModel进行4比特量化的Qwen2.5-Omni-7B模型。

• 量化位数: 4
• 动态量化: 未启用
• 分组大小: 128
• 激活描述: 启用
• 静态分组: 禁用
• 对称量化: 禁用
• 语言模型头部: 未量化
• 真顺序量化: 启用
• 量化方法: "gptq"
• 检查点格式: "gptq"
• 元信息:
  • 量化工具: gptqmodel:1.1.0
  • 工具链接: https://github.com/modelcloud/gptqmodel
  • 阻尼百分比: 0.1
  • 阻尼自动增量: 0.0015

模型大小

Qwen 2.5 Omni模型加载指南

安装步骤

根据Qwen官方文档，按以下步骤安装：

pip uninstall transformers  
pip install git+https://github.com/huggingface/transformers@3a1ead0aabed473eafe527915eea8c197d424356  
pip install accelerate  
pip install qwen-omni-utils[decord]  

从GitHub仓库安装GPTQModel

模型加载代码

import os  
import json  
import torch  
import torch.nn.functional as F  
import numpy as np  
from PIL import Image  
from typing import Any, Dict, List, Optional, Tuple, Union  

from transformers import (  
    Qwen2_5OmniModel,   
    Qwen2_5OmniProcessor,   
    AutoModelForVision2Seq,   
    AutoProcessor,   
    AutoTokenizer  
)  
from transformers.utils.hub import cached_file  
from transformers.generation.utils import GenerateOutput  

from gptqmodel import GPTQModel, QuantizeConfig, BACKEND  
from gptqmodel.models.base import BaseGPTQModel  
from gptqmodel.models.auto import MODEL_MAP, SUPPORTED_MODELS  
from gptqmodel.models._const import CPU  

from datasets import load_dataset  
from qwen_omni_utils import process_mm_info  

class Qwen25OmniThiknerGPTQ(BaseGPTQModel):  
    loader = Qwen2_5OmniModel  
    base_modules = [  
        "thinker.model.embed_tokens",   
        "thinker.model.norm",   
        "token2wav",   
        "thinker.audio_tower",   
        "thinker.model.rotary_emb",  
        "thinker.visual",   
        "talker"  
    ]  
    pre_lm_head_norm_module = "thinker.model.norm"  
    require_monkeypatch = False  
    layers_node = "thinker.model.layers"  
    layer_type = "Qwen2_5OmniDecoderLayer"  
    layer_modules = [  
        ["self_attn.k_proj", "self_attn.v_proj", "self_attn.q_proj"],  
        ["self_attn.o_proj"],  
        ["mlp.up_proj", "mlp.gate_proj"],  
        ["mlp.down_proj"],  
    ]  
   
    def pre_quantize_generate_hook_start(self):  
        self.thinker.visual = move_to(self.thinker.visual, device=self.quantize_config.device)  
        self.thinker.audio_tower = move_to(self.thinker.audio_tower, device=self.quantize_config.device)  

    def pre_quantize_generate_hook_end(self):  
        self.thinker.visual = move_to(self.thinker.visual, device=CPU)  
        self.thinker.audio_tower = move_to(self.thinker.audio_tower, device=CPU)  

    def preprocess_dataset(self, sample: Dict) -> Dict:  
        return sample  

MODEL_MAP["qwen2_5_omni"] = Qwen25OmniThiknerGPTQ  
SUPPORTED_MODELS.append("qwen2_5_omni")  

model_path = "/home/chentianqi/model/Qwen/Qwen2.5-Omni-7B-GPTQ-4bit"  

from types import MethodType  

@classmethod  
def patched_from_config(cls, config, *args, **kwargs):  
    kwargs.pop("trust_remote_code", None)  

    model = cls._from_config(config, **kwargs)  
    spk_path = cached_file(  
        model_path,  
        "spk_dict.pt",  
        subfolder=kwargs.pop("subfolder", None),  
        cache_dir=kwargs.pop("cache_dir", None),  
        force_download=kwargs.pop("force_download", False),  
        proxies=kwargs.pop("proxies", None),  
        resume_download=kwargs.pop("resume_download", None),  
        local_files_only=kwargs.pop("local_files_only", False),  
        token=kwargs.pop("use_auth_token", None),  
        revision=kwargs.pop("revision", None),  
    )  
    if spk_path is None:  
        raise ValueError(f"Speaker dictionary not found at {spk_path}")  

    model.load_speakers(spk_path)  
    return model  

Qwen2_5OmniModel.from_config = patched_from_config  

# 全精度模型加载  
# model = Qwen2_5OmniModel.from_pretrained(  
#     model_path,  
#     torch_dtype=torch.bfloat16,  
#     device_map="auto",  
#     attn_implementation="flash_attention_2",  
# )  

# GPTQ量化模型加载  
model = GPTQModel.load(  
    model_path,   
    device_map="cuda",   
    torch_dtype=torch.float16,     
    attn_implementation="flash_attention_2"  
)  

模型测试

from qwen_omni_utils import process_mm_info  
processor = Qwen2_5OmniProcessor.from_pretrained(model_path)  
# @title 推理函数  
def inference(video_path, prompt, sys_prompt):  
    messages = [  
        {"role": "system", "content": sys_prompt},  
        {"role": "user", "content": [  
                {"type": "text", "text": prompt},  
                {"type": "video", "video": video_path},  
            ]  
        },  
    ]  
    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)  
    audios, images, videos = process_mm_info(messages, use_audio_in_video=False)  
    inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors="pt", padding=True)  
    inputs = inputs.to(model.device).to(model.dtype)  

    output = model.generate(**inputs, use_audio_in_video=False, return_audio=False)  

    text = processor.batch_decode(output, skip_special_tokens=True, clean_up_tokenization_spaces=False)  
    return text  

video_path = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/screen.mp4"  
prompt = "请将论文摘要翻译成中文。"  

# 本地HuggingFace模型推理  
response = inference(video_path, prompt=prompt, sys_prompt="你是一个乐于助人的助手。")  
print(response[0])  

注意事项

• 代码中同时提供了全精度模型加载和GPTQ量化模型加载的选项（全精度部分已注释）