license: cc-by-nc-4.0 library_name: transformers pipeline_tag: video-text-to-text

视频多模态大语言模型的慢快架构

本仓库包含论文《视频多模态大语言模型的慢快架构》中提出的模型。

代码库: https://github.com/SHI-Labs/Slow-Fast-Video-Multimodal-LLM

简介

该模型采用创新的慢快架构来平衡视频理解中的时间分辨率和空间细节，克服了传统大语言模型的序列长度限制。它采用双令牌策略："快令牌"提供快速概览，而"慢令牌"通过交叉注意力机制实现指令感知的细节提取。

使用方法

import torch
import os
import numpy as np
from decord import VideoReader, cpu

from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
from llava.conversation import conv_templates, SeparatorStyle
from llava.model.builder import load_pretrained_model
from llava.mm_utils import tokenizer_image_token, get_model_name_from_path
from llava.utils import disable_torch_init

def load_video(video_path, max_frames_num):
        vr = VideoReader(video_path, num_threads=4)
        fps = round(vr.get_avg_fps())
        frame_idx = [i for i in range(0, len(vr), fps)]

        uniform_sampled_frames = np.linspace(0, len(vr) - 1, max_frames_num, dtype=int)
        frame_idx = uniform_sampled_frames.tolist()
        spare_frames = vr.get_batch(frame_idx).asnumpy()

        return spare_frames

# 模型加载
# 请确保已克隆代码仓库: git clone https://github.com/SHI-Labs/Slow-Fast-Video-Multimodal-LLM.git
model_path = "shi-labs/slowfast-video-mllm-qwen2-7b-convnext-576-frame64-s1t4" # 或其他检查点
video_path = "Slow-Fast-Video-Multimodal-LLM/assets/catinterrupt.mp4" # 从克隆仓库获取的示例视频路径
question = "请详细描述这个视频内容。"
max_frames=64 # 根据具体检查点设置

disable_torch_init()
model_path = os.path.expanduser(model_path)
model_name = get_model_name_from_path(model_path)
# 请确保传递trust_remote_code=True参数
tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, None, model_name, use_flash_attn=True, trust_remote_code=True)

if model.config.mm_use_im_start_end:
    prompt = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + "\n" + question
else:
    prompt = DEFAULT_IMAGE_TOKEN + "\n" + question

conv = conv_templates["qwen_1_5"].copy()
conv.append_message(conv.roles[0], prompt)
conv.append_message(conv.roles[1], None)
prompt = conv.get_prompt()

# 读取并处理视频
video = load_video(video_path, max_frames_num=max_frames)
video_tensor = image_processor.preprocess(video, return_tensors="pt")["pixel_values"].half().cuda()
videos = [video_tensor]

input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt')
input_ids = input_ids.to(device='cuda', non_blocking=True).unsqueeze(dim=0)

with torch.inference_mode():
    output_ids = model.generate(
        input_ids,
        images=videos,
        do_sample=True,
        max_new_tokens=1024,
        num_beams=1,
        temperature=0.2,
        top_p=1.0,
        use_cache=True)

outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
print(f"用户输入: {question}\n")
print(outputs)

引用文献

@misc{wang2025slowfast,
      title={视频多模态大语言模型的慢快架构},
      author={王浩天 and 杨正源 and 赵越 and 林斌 and 陈哲 and 曹越 and 杨红霞},
      year={2025},
      eprint={2504.01328},\
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2504.01328v1},
}