language: zh
tags:

• 视觉
• 文本生成图像
• 图像生成文本
• 多模态

Harmon：统一多模态理解与生成的视觉表征协调框架

协调视觉表征以实现统一多模态理解与生成

吴思泽、张文蔚、徐璐敏、金晟、吴中华、陶清怡、刘文涛、李玮、罗志强

简介

Harmon 是一种创新的统一多模态理解与生成框架。与现有先进架构采用不同编码器模型分离视觉理解和生成任务不同，该框架通过共享的MAR编码器协调理解与生成的视觉表征。Harmon在主流文本生成图像基准测试中展现出先进的生成性能，并在多模态理解任务中取得具有竞争力的结果。本代码库提供Harmon-0.5B和Harmon-1.5B两个模型变体，支持图像理解（图像到文本）和文本生成图像的推理代码。

模型变体	大语言模型	MAR编码器	Hugging Face模型库
Harmon-0.5B	Qwen2.5-0.5B-Instruct	MAR-Base
Harmon-1.5B	Qwen2.5-1.5B-Instruct	MAR-Huge

使用指南

🖌️ 图像到文本生成

import torch
import numpy as np
from transformers import AutoTokenizer, AutoModel
from einops import rearrange
from PIL import Image
import requests


PROMPT_TEMPLATE = dict(
    SYSTEM='<|im_start|>system\n{system}<|im_end|>\n',
    INSTRUCTION='<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n',
    SUFFIX='<|im_end|>',
    SUFFIX_AS_EOS=True,
    SEP='\n',
    STOP_WORDS=['<|im_end|>', '<|endoftext|>'])


def expand2square(pil_img, background_color):
    width, height = pil_img.size
    if width == height:
        return pil_img
    elif width > height:
        result = Image.new(pil_img.mode, (width, width), background_color)
        result.paste(pil_img, (0, (width - height) // 2))
        return result
    else:
        result = Image.new(pil_img.mode, (height, height), background_color)
        result.paste(pil_img, ((height - width) // 2, 0))
        return result


@torch.no_grad()
def question_answer(question,
                    image,
                    model,
                    tokenizer,
                    max_new_tokens=512,
                    image_size=512
                    ):
    assert image_size == 512
    image = expand2square(
        image, (127, 127, 127))
    image = image.resize(size=(image_size, image_size))
    image = torch.from_numpy(np.array(image)).to(dtype=model.dtype, device=model.device)
    image = rearrange(image, 'h w c -> c h w')[None]
    image = 2 * (image / 255) - 1

    prompt = PROMPT_TEMPLATE['INSTRUCTION'].format(input="<image>\n" + question)
    assert '<image>' in prompt
    image_length = (image_size // 16) ** 2 + model.mar.buffer_size
    prompt = prompt.replace('<image>', '<image>'*image_length)
    input_ids = tokenizer.encode(
        prompt, add_special_tokens=True, return_tensors='pt').cuda()
    _, z_enc = model.extract_visual_feature(model.encode(image))
    inputs_embeds = z_enc.new_zeros(*input_ids.shape, model.llm.config.hidden_size)
    inputs_embeds[input_ids == image_token_idx] = z_enc.flatten(0, 1)
    inputs_embeds[input_ids != image_token_idx] = model.llm.get_input_embeddings()(
        input_ids[input_ids != image_token_idx]
    )
    output = model.llm.generate(inputs_embeds=inputs_embeds,
                                use_cache=True,
                                do_sample=False,
                                max_new_tokens=max_new_tokens,
                                eos_token_id=tokenizer.eos_token_id,
                                pad_token_id=tokenizer.pad_token_id
                                if tokenizer.pad_token_id is not None else
                                tokenizer.eos_token_id
                                )
    return tokenizer.decode(output[0])


harmon_tokenizer = AutoTokenizer.from_pretrained("wusize/Harmon-1_5B",
                                                 trust_remote_code=True)
harmon_model = AutoModel.from_pretrained("wusize/Harmon-1_5B",
                                         trust_remote_code=True).eval().cuda().bfloat16()

special_tokens_dict = {'additional_special_tokens': ["<image>", ]}
num_added_toks = harmon_tokenizer.add_special_tokens(special_tokens_dict)
assert num_added_toks == 1

image_token_idx = harmon_tokenizer.encode("<image>", add_special_tokens=False)[-1]
print(f"图像标记: {harmon_tokenizer.decode(image_token_idx)}")

image_file = "http://images.cocodataset.org/val2017/000000039769.jpg"
raw_image = Image.open(requests.get(image_file, stream=True).raw).convert('RGB')

output_text = question_answer(question='详细描述这张图片。',
                              image=raw_image,
                              model=harmon_model,
                              tokenizer=harmon_tokenizer,
                              )

print(output_text)

🖼️ 文本生成图像

import os
import torch
from transformers import AutoTokenizer, AutoModel
from einops import rearrange
from PIL import Image


PROMPT_TEMPLATE = dict(
    SYSTEM='<|im_start|>system\n{system}<|im_end|>\n',
    INSTRUCTION='<|im_start|>user\n{input}<|im_end|>\n<|im_start|>assistant\n',
    SUFFIX='<|im_end|>',
    SUFFIX_AS_EOS=True,
    SEP='\n',
    STOP_WORDS=['<|im_end|>', '<|endoftext|>'])

GENERATION_TEMPLATE = "生成图像: {text}"


@torch.no_grad()
def generate_images(prompts,
                    negative_prompt,
                    tokenizer,
                    model,
                    output,
                    grid_size=2,   # 每个提示生成2x2图像
                    num_steps=64, cfg_scale=3.0, temperature=1.0, image_size=512):
    assert image_size == 512
    m = n = image_size // 16

    prompts = [
                  PROMPT_TEMPLATE['INSTRUCTION'].format(input=prompt)
                  for prompt in prompts
              ] * (grid_size ** 2)

    if cfg_scale != 1.0:
        prompts += [PROMPT_TEMPLATE['INSTRUCTION'].format(input=negative_prompt)] * len(prompts)

    inputs = tokenizer(
        prompts, add_special_tokens=True, return_tensors='pt', padding=True).to(model.device)

    images = model.sample(**inputs, num_iter=num_steps, cfg=cfg_scale, cfg_schedule="constant",
                          temperature=temperature, progress=True, image_shape=(m, n))
    images = rearrange(images, '(m n b) c h w -> b (m h) (n w) c', m=grid_size, n=grid_size)

    images = torch.clamp(
        127.5 * images + 128.0, 0, 255).to("cpu", dtype=torch.uint8).numpy()

    os.makedirs(output, exist_ok=True)
    for idx, image in enumerate(images):
        Image.fromarray(image).save(f"{output}/{idx:08d}.jpg")


harmon_tokenizer = AutoTokenizer.from_pretrained("wusize/Harmon-1_5B",
                                                 trust_remote_code=True)
harmon_model = AutoModel.from_pretrained("wusize/Harmon-1_5B",
                                         trust_remote_code=True).cuda().bfloat16().eval()


texts = ['左侧一只狗和右侧一只猫',
         '粉色停车标志的照片']
pos_prompts = [GENERATION_TEMPLATE.format(text=text) for text in texts]
neg_prompt = '生成图像'   # 用于分类器自由引导


generate_images(prompts=pos_prompts,
                negative_prompt=neg_prompt,
                tokenizer=harmon_tokenizer,
                model=harmon_model,
                output='output',)