import requests  
import torch  
from PIL import Image  
from transformers import AutoProcessor, AutoModelForCausalLM   

device = "cuda:0" if torch.cuda.is_available() else "cpu"  
torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32  

model = AutoModelForCausalLM.from_pretrained("microsoft/Florence-2-large", torch_dtype=torch_dtype, trust_remote_code=True).to(device)  
processor = AutoProcessor.from_pretrained("microsoft/Florence-2-large", trust_remote_code=True)  

url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true"  
image = Image.open(requests.get(url, stream=True).raw)  

def run_example(task_prompt, text_input=None):  
    if text_input is None:  
        prompt = task_prompt  
    else:  
        prompt = task_prompt + text_input  
    inputs = processor(text=prompt, images=image, return_tensors="pt").to(device, torch_dtype)  
    generated_ids = model.generate(  
      input_ids=inputs["input_ids"],  
      pixel_values=inputs["pixel_values"],  
      max_new_tokens=1024,  
      num_beams=3  
    )  
    generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0]  

    parsed_answer = processor.post_process_generation(generated_text, task=task_prompt, image_size=(image.width, image.height))  

    print(parsed_answer)  

图像描述

prompt = "<CAPTION>"  
run_example(prompt)  

详细描述

prompt = "<DETAILED_CAPTION>"  
run_example(prompt)  

更详细描述

prompt = "<MORE_DETAILED_CAPTION>"  
run_example(prompt)  

描述到短语定位

需额外输入描述文本，输出格式：
{'bboxes': [[坐标],...], 'labels': [...]}

task_prompt = "<CAPTION_TO_PHRASE_GROUNDING>"  
results = run_example(task_prompt, text_input="一辆绿色汽车停在黄色建筑前")  

目标检测

输出格式：
{'bboxes': [[坐标],...], 'labels': [...]}

prompt = "<OD>"  
run_example(prompt)  

密集区域描述

输出格式同目标检测

prompt = "<DENSE_REGION_CAPTION>"  
run_example(prompt)  

区域提议

输出格式同目标检测

prompt = "<REGION_PROPOSAL>"  
run_example(prompt)  

文字识别(OCR)

prompt = "<OCR>"  
run_example(prompt)  

带区域的OCR

输出格式：
{'quad_boxes': [[四边形坐标],...], 'labels': [...]}

prompt = "<OCR_WITH_REGION>"  
run_example(prompt)  

带置信度的目标检测

def run_example_with_score(task_prompt, text_input=None):  
    # 实现细节同上，增加置信度输出  
    ...  

prompt = "<OD>"  
run_example_with_score(prompt)  

更多示例详见Notebook