license: mit
library_name: transformers
pipeline_tag: image-to-text

[免安装！] 通过Hugging Face AutoModel快速启动

无需安装此GitHub仓库。请确保使用4.45.0版本的Transformers包（pip install transformers==4.45.0）。

使用q-sit进行图像质量解读对话：

import requests  
from PIL import Image  
import torch  
from transformers import AutoProcessor, LlavaOnevisionForConditionalGeneration  

model_id = "zhangzicheng/q-sit-mini"  
# 如需使用标准版，请切换为q-sit  
# model_id = "zhangzicheng/q-sit"  

model = LlavaOnevisionForConditionalGeneration.from_pretrained(  
    model_id,  
    torch_dtype=torch.float16,  
    low_cpu_mem_usage=True,  
).to(0)  

processor = AutoProcessor.from_pretrained(model_id)  

conversation = [  
    {  
        "role": "user",  
        "content": [  
            {"type": "text", "text": "这张图中人物的清晰度如何？"},  
            {"type": "image"},  
        ],  
    },  
]  
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)  

raw_image = Image.open(requests.get("https://github.com/Q-Future/Q-SiT/blob/main/44009500.jpg?raw=true",stream=True).raw)  

inputs = processor(images=raw_image, text=prompt, return_tensors='pt').to(0, torch.float16)  

output = model.generate(**inputs, max_new_tokens=200, do_sample=False)  
print(processor.decode(output[0][2:], skip_special_tokens=True).split("assistant")[-1])  
# 非常低  

使用q-sit进行图像质量评分：

import torch  
import requests  
from PIL import Image  
from transformers import AutoProcessor, LlavaOnevisionForConditionalGeneration, AutoTokenizer  
import numpy as np  

def wa5(logits):  
    logprobs = np.array([logits["Excellent"], logits["Good"], logits["Fair"], logits["Poor"], logits["Bad"]])  
    probs = np.exp(logprobs) / np.sum(np.exp(logprobs))  
    return np.inner(probs, np.array([1, 0.75, 0.5, 0.25, 0]))  

model_id = "zhangzicheng/q-sit-mini"  
model = LlavaOnevisionForConditionalGeneration.from_pretrained(  
    model_id,  
    torch_dtype=torch.float16,  
    low_cpu_mem_usage=True,  
).to(0)  

processor = AutoProcessor.from_pretrained(model_id)  
tokenizer = AutoTokenizer.from_pretrained(model_id)  

# 定义评分标签  
toks = ["Excellent", "Good", "Fair", "Poor", "Bad"]  
ids_ = [id_[0] for id_ in tokenizer(toks)["input_ids"]]  
print("评分标签ID:", ids_)  

conversation = [  
    {  
        "role": "user",  
        "content": [  
            {"type": "text", "text": "假设你是图像质量评估专家。\n请从以下五个等级中选择评分：Excellent（优）、Good（良）、Fair（中）、Poor（差）、Bad（劣）（从高到低排列）。\n你会如何评价这张图像的质量？"},  
            {"type": "image"},  
        ],  
    },  
]  
prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)  

# 加载图像  
raw_image = Image.open(requests.get("https://github.com/Q-Future/Q-SiT/blob/main/44009500.jpg?raw=true",stream=True).raw)  
inputs = processor(images=raw_image, text=prompt, return_tensors='pt').to(0, torch.float16)  

# 手动添加助手前缀"该图像质量为"  
prefix_text = "该图像质量为"  
prefix_ids = tokenizer(prefix_text, return_tensors="pt")["input_ids"].to(0)  
inputs["input_ids"] = torch.cat([inputs["input_ids"], prefix_ids], dim=-1)  
inputs["attention_mask"] = torch.ones_like(inputs["input_ids"])  # 更新注意力掩码  

# 仅生成一个评分标记  
output = model.generate(  
    **inputs,  
    max_new_tokens=1,  # 仅生成评分标记  
    output_logits=True,  
    return_dict_in_generate=True,  
)  

# 提取生成的评分标记logits  
last_logits = output.logits[-1][0]  # 形状: [vocab_size]  
logits_dict = {tok: last_logits[id_].item() for tok, id_ in zip(toks, ids_)}  
weighted_score = wa5(logits_dict)  
print("加权平均得分:", weighted_score)  
# 加权平均得分: 0.045549712192942585 （范围0-1）  
# 如需0-5分制，请乘以5  

如需在数据集上测试q-sit，请参考评估脚本。

该模型论文详见此处，代码库位于https://github.com/Q-Future/Q-SiT。