关于模型卡元数据的参考，请查阅规范：https://github.com/huggingface/hub-docs/blob/main/modelcard.md?plain=1

文档/指南：https://huggingface.co/docs/hub/model-cards

{}

Kosmos-2：将多模态大语言模型与现实世界锚定

此模型（Hub上的远程代码）已弃用。请使用 https://huggingface.co/microsoft/kosmos-2-patch14-224

输入格式有所变化：请参阅 https://huggingface.co/microsoft/kosmos-2-patch14-224 中的模型卡

（目前正致力于将Kosmos-2直接移植到transformers中。此仓库（远程代码）后续可能还需更多修复，包括破坏性变更。）

此Hub仓库包含微软原始Kosmos-2模型的HuggingFace transformers实现。

如何开始使用该模型

使用以下代码开始使用该模型。

import requests

from PIL import Image
from transformers import AutoProcessor, AutoModelForVision2Seq


model = AutoModelForVision2Seq.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)

prompt = "<grounding>一张"

url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.png"
image = Image.open(requests.get(url, stream=True).raw)

# 原始Kosmos-2演示会先保存图片再重新加载。对于某些图片，这会略微改变输入并影响生成结果。
# 如需匹配原始演示的输出，请取消以下两行注释。
# （例如演示中的`two_dogs.jpg`）
# image.save("new_image.jpg")
# image = Image.open("new_image.jpg")

inputs = processor(text=prompt, images=image, return_tensors="pt")

generated_ids = model.generate(
    pixel_values=inputs["pixel_values"],
    input_ids=inputs["input_ids"][:, :-1],
    attention_mask=inputs["attention_mask"][:, :-1],
    img_features=None,
    img_attn_mask=inputs["img_attn_mask"][:, :-1],
    use_cache=True,
    max_new_tokens=64,
)
generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]

# 设置`cleanup_and_extract=False`可查看原始模型生成内容。
processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)

print(processed_text)
# `<grounding> 一张<phrase>雪人</phrase><object><patch_index_0044><patch_index_0863></object>在<phrase>火堆</phrase><object><patch_index_0005><patch_index_0911></object>旁取暖的图片。`

# 默认情况下，生成文本会被清理并提取实体。
processed_text, entities = processor.post_process_generation(generated_text)

print(processed_text)
# `一张雪人在火堆旁取暖的图片。`

print(entities)
# `[('雪人', (3, 5), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('火堆', (8, 10), [(0.171875, 0.015625, 0.484375, 0.890625)])]`

在图片上绘制实体的边界框

获取entities后，可使用以下辅助函数在图片上绘制其边界框：

import cv2
import numpy as np
import os
import requests
import torch
import torchvision.transforms as T

from PIL import Image


def is_overlapping(rect1, rect2):
    x1, y1, x2, y2 = rect1
    x3, y3, x4, y4 = rect2
    return not (x2 < x3 or x1 > x4 or y2 < y3 or y1 > y4)


def draw_entity_boxes_on_image(image, entities, show=False, save_path=None):
    """_summary_
    参数:
        image (_type_): 图片或图片路径
        collect_entity_location (_type_): _description_
    """
    if isinstance(image, Image.Image):
        image_h = image.height
        image_w = image.width
        image = np.array(image)[:, :, [2, 1, 0]]
    elif isinstance(image, str):
        if os.path.exists(image):
            pil_img = Image.open(image).convert("RGB")
            image = np.array(pil_img)[:, :, [2, 1, 0]]
            image_h = pil_img.height
            image_w = pil_img.width
        else:
            raise ValueError(f"无效图片路径, {image}")
    elif isinstance(image, torch.Tensor):
        image_tensor = image.cpu()
        reverse_norm_mean = torch.tensor([0.48145466, 0.4578275, 0.40821073])[:, None, None]
        reverse_norm_std = torch.tensor([0.26862954, 0.26130258, 0.27577711])[:, None, None]
        image_tensor = image_tensor * reverse_norm_std + reverse_norm_mean
        pil_img = T.ToPILImage()(image_tensor)
        image_h = pil_img.height
        image_w = pil_img.width
        image = np.array(pil_img)[:, :, [2, 1, 0]]
    else:
        raise ValueError(f"无效图片格式, {type(image)} for {image}")

    if len(entities) == 0:
        return image

    new_image = image.copy()
    previous_bboxes = []
    text_size = 1
    text_line = 1
    box_line = 3
    (c_width, text_height), _ = cv2.getTextSize("F", cv2.FONT_HERSHEY_COMPLEX, text_size, text_line)
    base_height = int(text_height * 0.675)
    text_offset_original = text_height - base_height
    text_spaces = 3

    for entity_name, (start, end), bboxes in entities:
        for (x1_norm, y1_norm, x2_norm, y2_norm) in bboxes:
            orig_x1, orig_y1, orig_x2, orig_y2 = int(x1_norm * image_w), int(y1_norm * image_h), int(x2_norm * image_w), int(y2_norm * image_h)
            color = tuple(np.random.randint(0, 255, size=3).tolist())
            new_image = cv2.rectangle(new_image, (orig_x1, orig_y1), (orig_x2, orig_y2), color, box_line)

            l_o, r_o = box_line // 2 + box_line % 2, box_line // 2 + box_line % 2 + 1

            x1 = orig_x1 - l_o
            y1 = orig_y1 - l_o

            if y1 < text_height + text_offset_original + 2 * text_spaces:
                y1 = orig_y1 + r_o + text_height + text_offset_original + 2 * text_spaces
                x1 = orig_x1 + r_o

            (text_width, text_height), _ = cv2.getTextSize(f"  {entity_name}", cv2.FONT_HERSHEY_COMPLEX, text_size, text_line)
            text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2 = x1, y1 - (text_height + text_offset_original + 2 * text_spaces), x1 + text_width, y1

            for prev_bbox in previous_bboxes:
                while is_overlapping((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2), prev_bbox):
                    text_bg_y1 += (text_height + text_offset_original + 2 * text_spaces)
                    text_bg_y2 += (text_height + text_offset_original + 2 * text_spaces)
                    y1 += (text_height + text_offset_original + 2 * text_spaces)

                    if text_bg_y2 >= image_h:
                        text_bg_y1 = max(0, image_h - (text_height + text_offset_original + 2 * text_spaces))
                        text_bg_y2 = image_h
                        y1 = image_h
                        break

            alpha = 0.5
            for i in range(text_bg_y1, text_bg_y2):
                for j in range(text_bg_x1, text_bg_x2):
                    if i < image_h and j < image_w:
                        if j < text_bg_x1 + 1.35 * c_width:
                            bg_color = color
                        else:
                            bg_color = [255, 255, 255]
                        new_image[i, j] = (alpha * new_image[i, j] + (1 - alpha) * np.array(bg_color)).astype(np.uint8)

            cv2.putText(
                new_image, f"  {entity_name}", (x1, y1 - text_offset_original - 1 * text_spaces), cv2.FONT_HERSHEY_COMPLEX, text_size, (0, 0, 0), text_line, cv2.LINE_AA
            )
            previous_bboxes.append((text_bg_x1, text_bg_y1, text_bg_x2, text_bg_y2))

    pil_image = Image.fromarray(new_image[:, :, [2, 1, 0]])
    if save_path:
        pil_image.save(save_path)
    if show:
        pil_image.show()

    return new_image


url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.jpg"
image = Image.open(requests.get(url, stream=True).raw)

entities = [('雪人', (3, 5), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('火堆', (8, 10), [(0.171875, 0.015625, 0.484375, 0.890625)])]

draw_entity_boxes_on_image(image, entities, show=True)

以下是标注后的图片：

任务

该模型可通过改变提示词执行不同任务。

首先，定义一个运行提示词的函数。

import requests

from PIL import Image
from transformers import AutoProcessor, AutoModelForVision2Seq


model = AutoModelForVision2Seq.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)

url = "https://huggingface.co/ydshieh/kosmos-2-patch14-224/resolve/main/snowman.png"
image = Image.open(requests.get(url, stream=True).raw)

def run_example(prompt):
    inputs = processor(text=prompt, images=image, return_tensors="pt")
    generated_ids = model.generate(
        pixel_values=inputs["pixel_values"],
        input_ids=inputs["input_ids"][:, :-1],
        attention_mask=inputs["attention_mask"][:, :-1],
        img_features=None,
        img_attn_mask=inputs["img_attn_mask"][:, :-1],
        use_cache=True,
        max_new_tokens=64,
    )
    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
    _processed_text = processor.post_process_generation(generated_text, cleanup_and_extract=False)
    processed_text, entities = processor.post_process_generation(generated_text)
    print(processed_text)
    print(entities)
    print(_processed_text)

以下是Kosmos-2能执行的任务：

多模态锚定

• 短语锚定

prompt = "<grounding><phrase>雪人</phrase>"
run_example(prompt)

# 雪人正在火堆旁取暖
# [('雪人', (0, 2), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('火堆', (6, 8), [(0.203125, 0.015625, 0.453125, 0.859375)])]

# <grounding><phrase>雪人</phrase><object><patch_index_0044><patch_index_0863></object>正在<phrase>火堆</phrase><object><patch_index_0006><patch_index_0878></object>旁取暖

• 指代表达理解

prompt = "<grounding><phrase>火堆旁的雪人</phrase>"
run_example(prompt)

# 火堆旁的雪人
# [('火堆旁的雪人', (0, 5), [(0.390625, 0.046875, 0.984375, 0.828125)])]

# <grounding><phrase>火堆旁的雪人</phrase><object><patch_index_0044><patch_index_0863></object>

多模态指代

• 指代表达生成

prompt = "<grounding><phrase>它</phrase><object><patch_index_0044><patch_index_0863></object>是"
run_example(prompt)

# 它是戴着帽子和围巾的雪人
# [('它', (0, 1), [(0.390625, 0.046875, 0.984375, 0.828125)])]

# <grounding><phrase>它</phrase><object><patch_index_0044><patch_index_0863></object>是戴着帽子和围巾的雪人

感知-语言任务

• 锚定视觉问答

prompt = "<grounding>问题：这张图片有什么特别之处？回答："
run_example(prompt)

# 问题：这张图片有什么特别之处？回答：图片中有一个雪人坐在雪地里的篝火旁。
# [('雪人', (24, 26), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('篝火', (33, 35), [(0.109375, 0.640625, 0.546875, 0.984375)])]

# <grounding>问题：这张图片有什么特别之处？回答：图片中有<phrase>雪人</phrase><object><patch_index_0044><patch_index_0863></object>坐在<phrase>篝火</phrase><object><patch_index_0643><patch_index_1009></object>旁的雪地里。

• 通过边界框进行多模态指代的锚定视觉问答

prompt = "<grounding>问题：<phrase>火堆</phrase><object><patch_index_0005><patch_index_0911></object>旁边是什么？回答："
run_example(prompt)

# 问题：火堆旁边是什么？回答：雪人旁边。
# [('火堆', (3, 5), [(0.171875, 0.015625, 0.484375, 0.890625)]), ('雪人', (14, 16), [(0.390625, 0.046875, 0.984375, 0.828125)])]

# <grounding>问题：<phrase>火堆</phrase><object><patch_index_0005><patch_index_0911></object>旁边是什么？回答：<phrase>雪人</phrase><object><patch_index_0044><patch_index_0863></object>旁边。

锚定图片描述

• 简洁描述

prompt = "<grounding>一张"
run_example(prompt)

# 一张雪人在篝火旁取暖的图片。
# [('雪人', (2, 4), [(0.390625, 0.046875, 0.984375, 0.828125)]), ('篝火', (7, 9), [(0.109375, 0.640625, 0.546875, 0.984375)])]

# <grounding>一张<phrase>雪人</phrase><object><patch_index_0044><patch_index_0863></object>在<phrase>篝火</phrase><object><patch_index_0643><patch_index_1009></object>旁取暖的图片。

• 详细描述

prompt = "<grounding>详细描述这张图片："
run_example(prompt)

# 详细描述这张图片：图片中有一个雪人坐在雪地里的篝火旁。它戴着帽子、围巾和手套，附近有一个锅和一个杯子
# [('篝火', (24, 26), [(0.171875, 0.015625, 0.484375, 0.984375)]), ('帽子', (32, 34), [(0.515625, 0.046875, 0.828125, 0.234375)]), ('围巾', (36, 38), [(0.515625, 0.234375, 0.890625, 0.578125)]), ('手套', (42, 44), [(0.515625, 0.390625, 0.640625, 0.515625)]), ('锅', (50, 51), [(0.078125, 0.609375, 0.265625, 0.859375)])]

# <grounding>详细描述这张图片：图片中有<phrase>雪人</phrase><object>坐在<phrase>篝火</phrase><object><patch_index_0005><patch_index_1007></object>旁的雪地里。它戴着<phrase>帽子</phrase><object><patch_index_0048><patch_index_0250></object>、<phrase>围巾</phrase><object><patch_index_0240><patch_index_0604></object>和<phrase>手套</phrase><object><patch_index_0400><patch_index_0532></object>，附近有<phrase>锅</phrase><object><patch_index_0610><patch_index_0872></object>和<phrase>杯子</phrase><object>

运行Flask服务器

_flask_kosmos2.py_展示了该模型的Flask服务器实现。 它允许通过REST API访问模型。

启动服务器后，可向http://localhost:8005/process_prompt发送POST请求，包含以下表单数据：

• prompt：例如<grounding>一张
• image：作为二进制数据的图片文件

响应将返回以下JSON格式：

• message：Kosmos-2生成的文本
• entities：提取的实体

可通过Postman等应用测试。确保图片字段设置为File。

from PIL import Image
from transformers import AutoProcessor, AutoModelForVision2Seq
from flask import Flask, request, jsonify
import json

app = Flask(__name__)

model = AutoModelForVision2Seq.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)
processor = AutoProcessor.from_pretrained("ydshieh/kosmos-2-patch14-224", trust_remote_code=True)


@app.route('/process_prompt', methods=['POST'])
def process_prompt():
    try:
        uploaded_file = request.files['image']
        prompt = request.form.get('prompt')
        image = Image.open(uploaded_file.stream)

        print(image.size)

        inputs = processor(text=prompt, images=image, return_tensors="pt")

        generated_ids = model.generate(
            pixel_values=inputs["pixel_values"],
            input_ids=inputs["input_ids"][:, :-1],
            attention_mask=inputs["attention_mask"][:, :-1],
            img_features=None,
            img_attn_mask=inputs["img_attn_mask"][:, :-1],
            use_cache=True,
            max_new_tokens=64,
        )
        generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]

        processed_text, entities = processor.post_process_generation(generated_text)
        parsed_entities = entities_to_json(entities)
        print(generated_text)
        print(processed_text)
        return jsonify({"message": processed_text, 'entities': parsed_entities})
    except Exception as e:
        return jsonify({"error": str(e)})


def entities_to_json(entities):
    result = []
    for e in entities:
        label = e[0]
        box_coords = e[1]
        box_size = e[2][0]
        entity_result = {
            "label": label,
            "boundingBoxPosition": {"x": box_coords[0], "y": box_coords[1]},
            "boundingBox": {"x_min": box_size[0], "y_min": box_size[1], "x_max": box_size[2], "y_max": box_size[3]}
        }
        print(entity_result)
        result.append(entity_result)

    return result


if __name__ == '__main__':
    app.run(host='localhost', port=8005)